To install
pkg install openstack evs swift swiftclient pkg:/install/installadm mysql-56 mysql-56/client
First configure MySQL 5.6
zfs create -o mountpoint=/mysql zones/mysql mkdir /mysql/logs mkdir /mysql/innodb cp /etc/mysql/5.6/my.cnf /mysql/ cd /etc/mysql/5.6/; mv my.cnf my.cnf.orig; ln -s /mysql/my.cnf chown -R mysql:mysql /mysql
sample my.cnf
# For advice on how to change settings please see # http://dev.mysql.com/doc/refman/5.6/en/server-configuration-defaults.html [mysqld] default-storage-engine = InnoDB collation-server = utf8_unicode_ci init-connect = 'SET NAMES utf8' character-set-server = utf8 # Remove leading # and set to the amount of RAM for the most important data # cache in MySQL. Start at 70% of total RAM for dedicated server, else 10%. # innodb_buffer_pool_size = 128M # Remove leading # to turn on a very important data integrity option: logging # changes to the binary log between backups. # log_bin # These are commonly set, remove the # and set as required. basedir = /usr/mysql/5.6 datadir = /mysql/innodb port = 3306 # server_id = ..... socket = /tmp/mysql.sock # Remove leading # to set options mainly useful for reporting servers. # The server defaults are faster for transactions and fast SELECTs. # Adjust sizes as needed, experiment to find the optimal values. join_buffer_size = 128M sort_buffer_size = 2M read_rnd_buffer_size = 2M ### Copied from a 5.5 config innodb_data_home_dir = /mysql/innodb innodb_data_file_path = ibdata1:10M:autoextend innodb_log_group_home_dir = /mysql/innodb # You can set .._buffer_pool_size up to 50 – 80 % # of RAM but beware of setting memory usage too high innodb_buffer_pool_size = 16M #innodb_additional_mem_pool_size = 2M # Set .._log_file_size to 25 % of buffer pool size innodb_log_file_size = 5M innodb_log_buffer_size = 8M innodb_flush_log_at_trx_commit = 1 innodb_lock_wait_timeout = 50 sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES [client] default-character-set=utf8 [mysql] default-character-set=utf8
Now lets create a MySQL DB
export PATH=$PATH:/usr/mysql/5.6/bin /usr/mysql/5.6/bin/mysql_install_db --defaults-file=/mysql/my.cnf --user=mysql --datadir=/mysql/innodb --basedir=/usr/mysql/5.6 --pid-file=/mysql/mysql.pid
Now start mysql
/usr/mysql/5.6/bin/mysqld --defaults-file=/mysql/my.cnf --basedir=/usr/mysql/5.6 --datadir=/mysql/innodb --user=mysql --pid-file=/mysql/mysql.pid &
Update root password
mysqladmin -u root password [curent root password]
Secure MySQL
/usr/mysql/5.6/bin/mysql_secure_installation Enter current password for root (enter for none): Set root password? [Y/n] y Remove anonymous users? [Y/n] y Disallow root login remotely? [Y/n] n Remove test database and access to it? [Y/n] y Reload privilege tables now? [Y/n] y
Configure SMF
svccfg -s svc:/application/database/mysql:version_56 setprop mysql/cnf=/mysql/my.cnf svccfg -s svc:/application/database/mysql:version_56 setprop mysql/data=/mysql/innodb svcadm refresh mysql:version_56
Stop the MySQL instance running from before and start with SMF
kill %1 svcadm enable mysql
Configure rabitMQ
Create rabbitmq files in /etc/rabbitmq
#rabbitmq-env.conf
NODENAME=rabbit@os1
NODE_IP_ADDRESS=10.10.3.121
CONFIG_FILE=/etc/rabbitmq
#rabbitmq.config
[{rabbit,
[{cluster_nodes, {[‘rabbit@os1’], disc}}]}].
Enable rabitMQ
chown -R rabbitmq:bin /etc/rabbitmq svcadm enable svc:/application/rabbitmq:default
You have to add/update a rabbitmq admin user, guest user is dissallowed.
rabbitmqctl set_policy HA '^(?!amq.).*' '{"ha-mode": "all"}'
rabbitmqctl add_user admin password
rabbitmqctl set_permissions admin ".*" ".*" ".*"
svcadm restart rad:local
Update MySQL with all user data
mysql -u root -p drop database if exists nova; drop database if exists cinder; drop database if exists glance; drop database if exists keystone; drop database if exists neutron; drop database if exists heat; create database cinder; default character set utf8 default collate utf8_general_ci; grant all privileges on cinder.* to 'cinder'@'os1' identified by 'password'; grant all privileges on cinder.* to 'cinder'@'os1' identified by 'password'; create database glance; default character set utf8 default collate utf8_general_ci; grant all privileges on glance.* to 'glance'@'os1' identified by 'password'; create database keystone; default character set utf8 default collate utf8_general_ci; grant all privileges on keystone.* to 'keystone'@'os1' identified by 'password'; create database nova; default character set utf8 default collate utf8_general_ci; grant all privileges on nova.* to 'nova'@'os1' identified by 'password'; create database neutron; default character set utf8 default collate utf8_general_ci; grant all privileges on neutron.* to 'neutron'@'os1' identified by 'password'; create database heat default character set utf8 default collate utf8_general_ci; grant all privileges on heat.* to 'heat'@'os1' identified by 'password'; grant all privileges on keystone.* to 'keystone'@'localhost' identified by 'password'; grant all privileges on heat.* to 'heat'@'localhost' identified by 'password'; grant all privileges on cinder.* to 'cinder'@'localhost' identified by 'password'; grant all privileges on glance.* to 'glance'@'localhost' identified by 'password'; grant all privileges on neutron.* to 'neutron'@'localhost' identified by 'password'; grant all privileges on nova.* to 'nova'@'localhost' identified by 'password'; grant all privileges on keystone.* to 'keystone'@'%' identified by 'password'; grant all privileges on heat.* to 'heat'@'%' identified by 'password'; grant all privileges on cinder.* to 'cinder'@'%' identified by 'password'; grant all privileges on glance.* to 'glance'@'%' identified by 'password'; grant all privileges on neutron.* to 'neutron'@'%' identified by 'password'; grant all privileges on nova.* to 'nova'@'%' identified by 'password'; flush privileges; quit
To verify utf8 on db
mysql>show create database mysql;
Configure keystone
cp /usr/demo/openstack/keystone/sample_data.sh /usr/demo/openstack/keystone/sample_data.sh-org
Modify keystone
/etc/keystone/keystone.conf admin_token ADMIN – replace with 9d77f9bad250d97c365e (openssl rand -hex 10) public_bind_host ip address (10.10.3.121) admin_bind_host ip address (10.10.3.121) rabbit_host ip address 10.10.3.121 qpid_hostname ip address 10.10.3.121 connection=mysql://keystone:password@localhost/keystone su - keystone echo "export SERVICE_TOKEN=e541142067a0e0a7704e" >>.profile
Modify sample_data.sh before running
Add a set -x for debugging
#Add a line on top with comment password
ADMIN_PASSWORD=${ADMIN_PASSWORD:-secrete}
With
ADMIN_PASSWORD=${ADMIN_PASSWORD:-password}
Change localhost to public name (os1) - replace with real hostname
#replace DEMO_TENANT with devtech101
#replace like the below
export OS_AUTH_URL=http://os1:5000/v2.0
export OS_PASSWORD="${ADMIN_PASSWORD}"
export OS_TENANT_NAME=service
export OS_USERNAME=admin
Before start keystone
su - keystone /usr/bin/keystone-manage db_sync
Enable keystone
svcadm enable keystone su - keystone ### is depreciated /usr/bin/keystone-manage pki_setup # Add to keystone .profile export SERVICE_ENDPOINT=http://os1:35357/v2.0 export SERVICE_TOKEN=9d77f9bad250d97c365e (openssl rand -hex 10 from above keystone.profile)
now su to keystone and run
su - keystone /usr/demo/openstack/keystone/sample_data.sh
How to configure Glance
/etc/glance/glance-api.conf
[DEFAULT] registry_host = 10.10.3.121 admin_user = glance admin_password = password admin_tenant_name = service auth_url = http://10.10.3.121:5000/v2.0 auth_strategy = keystone default_publisher_id = image.os1.domain.com rabbit_host = 10.10.3.121 rabbit_userid = admin rabbit_password = password qpid_hostname = os1 [database] connection=mysql://glance:password@localhost/glance [keystone_authtoken] auth_uri = http://10.10.3.121:5000/v2.0/ identity_uri = http://10.10.3.121:35357/ admin_tenant_name = service admin_user = glance admin_password = password signing_dir = /var/lib/glance/keystone-signing
/etc/glance/glance-cache.conf
[DEFAULT] registry_host = 10.10.3.121 auth_url = http://10.10.3.1211:5000/v2.0/ identity_uri = http://10.10.3.1211:35357/ admin_tenant_name = service admin_user = glance admin_password = password
/etc/glance/glance-registry.conf
[DEFAULT] bind_host = 10.10.3.121 default_publisher_id = image.os1.domain.com rabbit_host = 10.10.3.121 rabbit_userid = admin rabbit_password = password qpid_hostname = 10.10.3.121 [database] connection = mysql://glance:password@localhost/glance [keystone_authtoken] auth_uri = http://10.10.3.121:5000/v2.0/ identity_uri = http://10.10.3.121:35357/ admin_tenant_name = service admin_user = glance admin_password = password
/etc/glance/glance-api-paste.ini
[filter:authtoken] auth_uri = http://10.10.3.121:5000/v2.0/ identity_uri = http://10.10.3.121:35357/ admin_tenant_name = service admin_user = glance admin_password = password
/etc/glance/glance-registry-paste.ini
[filter:authtoken] auth_uri = http://10.10.3.121:5000/v2.0/ identity_uri = http://10.10.3.121:35357/ admin_tenant_name = service admin_user = glance admin_password = password
/etc/glance/glance-scrubber.conf
[DEFAULT] registry_host = 10.10.3.121 auth_url = http://10.10.3.121:5000/v2.0/ identity_uri = http://10.10.3.121:35357/ admin_tenant_name = service admin_user = glance admin_password = password [database] connection=mysql://glance:password@localhost/glance
Now enable all services
svcadm enable -rs glance-api glance-db glance-registry glance-scrubber
Configuring Nova
[DEFAULT] qpid_hostname=10.10.3.121 rabbit_host=10.10.3.121 rabbit_userid=admin rabbit_password=password my_ip=10.10.3.121 host=os1 firewall_driver=nova.virt.firewall.NoopFirewallDriver [database] connection=mysql://nova:password@localhost/nova [glance] host=10.10.3.121 [keystone_authtoken] auth_uri=http://10.10.3.121:5000/v2.0/ identity_uri=http://10.10.3.121:35357/ admin_user=nova admin_password=password admin_tenant_name=service [neutron] url=http://10.10.3.121:9696 admin_username=neutron admin_password=password admin_tenant_id=service admin_auth_url=http://10.10.3.121:5000/v2.0
Enable the Nova SMF services.
svcadm enable -rs nova-conductor svcadm enable -rs nova-api-osapi-compute nova-cert nova-scheduler
Create .profile for nova
export OS_USERNAME=nova export OS_PASSWORD=password export OS_TENANT_NAME=service export OS_AUTH_URL=http://10.10.3.121:5000/v2.0/
Installing Horizon
Modify /etc/openstack_dashboard/local_settings.py
gsed -i -e s@SECURE_PROXY_SSL_HEADER@#SECURE_PROXY_SSL_HEADER@ -e s@CSRF_COOKIE_SECURE@#CSRF_COOKIE_SECURE@ -e s@SESSION_COOKIE_SECURE@#SESSION_COOKIE_SECURE@ /etc/openstack_dashboard/local_settings.py
Configure certifcates
openssl req -new -x509 -nodes -out horizon.crt -keyout horizon.key Generating a 1024 bit RSA private key ..++++++ .++++++ writing new private key to 'horizon.key' ----- You are about to be asked to enter information that will be incorporated into your certificate request. What you are about to enter is what is called a Distinguished Name or a DN. There are quite a few fields but you can leave some blank For some fields there will be a default value, If you enter '.', the field will be left blank. ----- Country Name (2 letter code) []:US State or Province Name (full name) []:New-York Locality Name (eg, city) []:NY Organization Name (eg, company) []:Devtech101 Organizational Unit Name (eg, section) []:IS Common Name (e.g. server FQDN or YOUR name) []:os1.domain.com Email Address []:admin@devtech101.com
Move in to place
chown webservd:webservd horizon.* chmod 0644 horizon.* mv horizon.* /etc/openstack_dashboard
Configure horizon web server
cp /etc/apache2/2.4/samples-conf.d/openstack-dashboard-http.conf /etc/apache2/2.4/conf.d/ sed -e "/SSLCertificateFile/s:/path.*:/etc/openstack_dashboard/horizon.crt:" -e "/SSLCACertificateFile/d" -e "/SSLCertificateKeyFile/s:/path.*:/etc/openstack_dashboard/horizon.key:" < /etc/apache2/2.4/samples-conf.d/openstack-dashboard-tls.conf > /etc/apache2/2.4/conf.d/openstack-dashboard-tls.conf # change hostname in openstack-dashboard-tls.conf RedirectPermanent /horizon https://os1.domain.com/horizon ServerName os1.domain.com
Horizon bugs
# Do not copy samples-conf.d/openstack-dashboard-tls.conf to conf.d/ chown -R webservd /usr/lib/python2.7/vendor-packages/openstack_dashboard/static # Attaching a ZFS volume dose not get attached unless you reboot the zone. i .e. you can do manual zonecfg apply will work # Removing attached volume works but levees drive offline instead of cleaning # Adding more then one iqn to zfssa_initiator will not work and break (not according to spec)
Configure Cinder
/etc/nova/cinder.conf
[DEFAULT] qpid_hostname=10.10.3.121 rabbit_host=10.10.3.121 rabbit_userid=admin rabbit_password=password my_ip=10.10.3.121 [database] connection=mysql://cinder:password@localhost/cinder [keystone_authtoken] auth_uri=http://10.10.3.121:5000/v2.0/ identity_uri=http://10.10.3.121:35357/ admin_user=cinder admin_password=password admin_tenant_name=service
If iSCSI targets are configured, enable the corresponding SMF services
svcadm enable iscsi/target stmf
Enable the Cinder SMF services
svcadm enable -rs cinder-db svcadm enable -rs cinder-api cinder-scheduler
How to Configure the ZFS Storage Appliance iSCSI Cinder
Driver
Coming soon
Configure Neutron
qpid_hostname=10.10.3.121 bind_host = 10.10.3.121 rabbit_host=10.10.3.121 rabbit_userid=admin rabbit_password=password host=10.10.3.121 [keystone_authtoken] auth_uri = http://10.10.3.121:5000/v2.0/ identity_uri = http://10.10.3.121:35357/ admin_tenant_name = service admin_user = neutron admin_password = password [database] connection = mysql://neutron:password@localhost/neutron
Modify also these files
/etc/neutron/plugins/evs/evs_plugin.ini evs_controller = ssh://evsuser@os1 # /etc/neutron/dhcp_agent.ini evs_controller = ssh://evsuser@os1
Configure EVS
su - evsuser -c "ssh-keygen -N '' -f /var/user/evsuser/.ssh/id_rsa -t rsa" su - neutron -c "ssh-keygen -N '' -f /var/lib/neutron/.ssh/id_rsa -t rsa" ssh-keygen -N '' -f /root/.ssh/id_rsa -t rsa
Combine SSH key’s
cat /var/user/evsuser/.ssh/id_rsa.pub /var/lib/neutron/.ssh/id_rsa.pub /root/.ssh/id_rsa.pub >> /var/user/evsuser/.ssh/authorized_keys
Enable keys
su - evsuser -c "ssh evsuser@os1 true" su - neutron -c "ssh evsuser@os1 true" ssh evsuser@os1 true su - evsuser -c "ssh evsuser@localhost true" su - neutron -c "ssh evsuser@localhost true" ssh evsuser@localhost true
Configure the elastic virtual switch
evsadm set-prop -p controller=ssh://evsuser@localhost evsadm set-controlprop -p l2-type=vlan evsadm set-controlprop -p uplink-port=aggr1 evsadm set-controlprop -p vlan-range=2000-3900 # evsadm show-controlprop ipadm set-prop -p forwarding=on ipv4
Start the IP Filter service
svcadm enable -rs ipfilter
Enable the Neutron server service
svcadm enable -rs neutron-server neutron-dhcp-agent
Configuring Compute node
pkg install openstack
add to /etc/nova/nova.conf
[DEFAULT] rabbit_host=10.10.3.121 rabbit_userid=admin rabbit_password=password my_ip=10.10.3.121 host=os2 firewall_driver=nova.virt.firewall.NoopFirewallDriver [database] connection=mysql://nova:password@localhost/nova [glance] host=10.10.3.121 keystone_ec2_url=http://10.10.3.121:5000/v2.0/ec2tokens [keystone_authtoken] auth_uri=http://10.10.3.121:5000/v2.0/ identity_uri=http://10.10.3.121:35357/ admin_user=nova admin_password=password admin_tenant_name=service [neutron] url=http://10.10.3.121:9696 admin_username=neutron admin_password=password admin_tenant_id=service admin_auth_url=http://10.10.3.121:5000/v2.0
svcadm restart rad:local
Configure EVS
evsadm set-prop -p controller=ssh://evsuser@10.10.3.121 su - root -c "ssh-keygen -N '' -f /root/.ssh/id_rsa -t rsa" cat /root/.ssh/id_rsa.pub cat [remote_compute]/id_rsa.pub >> /var/user/evsuser/.ssh/authorized_keys ssh evsuser@10.10.3.121 true
Enable nova compute
svcadm enable nova-compute
How to Enable Console Access – on each compute node
/etc/nova/nova.conf
[DEFAULT] vnc_enabled = true vncserver_listen = 10.10.3.121 novncproxy_port = 6080 novncproxy_base_url =http://10.10.3.121:6080/vnc_auto.html novncproxy_host = 10.10.3.121 # For private network vnc_enabled = true vncserver_listen = internal-IP novncproxy_port=6080 novncproxy_base_url = http://public-IP:6080/vnc_auto.html vncserver_proxyclient_address = internal-IP
Enable the nova-novncproxy service
svcadm enable nova-novncproxy svcadm restart nova-compute # or enable
On the Controller node
If the Compute node’s IP addresses are accessible from the public facing
network
svcadm enable nova-consoleauth
Note: If the Compute node is in a private network Set the following parameters
[DEFAULT] novncproxy_base_url=http://public-IP:6080/vnc_auto.html svcadm enable nova-consoleauth svcadm enable nova-novncproxy
Configuring the Storage Node
/etc/cinder/cinder.conf
[DEFAULT]
san_is_local=true
rabbit_host=10.10.3.121
rabbit_userid=admin
rabbit_password=password
my_ip=10.10.3.122
glance_host=10.10.3.121
zfs_volume_base=cinder/cinder
[database]
connection=mysql://cinder:password@10.10.3.121/cinder
[keystone_authtoken]
auth_uri=http://10.10.3.121:5000/v2.0/
identity_uri=http://10.10.3.121:35357/
admin_user=cinder
admin_password=password
admin_tenant_name=service
[DEFAULT] san_is_local=true rabbit_host=10.10.3.121 rabbit_userid=admin rabbit_password=password my_ip=10.10.3.122 glance_host=10.10.3.121 zfs_volume_base=cinder/cinder [database] connection=mysql://cinder:password@10.10.3.121/cinder [keystone_authtoken] auth_uri=http://10.10.3.121:5000/v2.0/ identity_uri=http://10.10.3.121:35357/ admin_user=cinder admin_password=password admin_tenant_name=service
Start cinder srevice
svcadm enable -rs cinder-db cinder-volume:default cinder-volume:setup svcadm enable -rs iscsi/target