Production upgrade steps

marko-k0 · marko-k0 · commit 9707488da585 · 2017-11-24T11:38:16.000+01:00
diff --git a/04-lbaas-upgrade/haproxy_proxies.j2.patch b/04-lbaas-upgrade/haproxy_proxies.j2.patch
diff --git a/04-lbaas-upgrade/lbaas-migration.sh b/04-lbaas-upgrade/lbaas-migration.sh
diff --git a/04-lbaas-upgrade/migration_init/0-source-me.sh b/04-lbaas-upgrade/migration_init/0-source-me.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export LBAAS_HOSTS=$(sudo mysql neutron -sNe 'select host from agents where `binary` = "neutron-lbaas-agent" and TIMESTAMPDIFF(SECOND, heartbeat_timestamp, NOw()) < 300' | awk 1 ORS=',')
+export LBAAS_HOSTS_COUNT=$(sudo mysql neutron -sNe 'select count(*) from agents where `binary` = "neutron-lbaas-agent" and TIMESTAMPDIFF(SECOND, heartbeat_timestamp, NOw()) < 300')
diff --git a/04-lbaas-upgrade/migration_init/1-clean-healthmonitors.sh b/04-lbaas-upgrade/migration_init/1-clean-healthmonitors.sh
@@ -0,0 +1,4 @@
+# "clean' pools so that they have at most 1 health monitor
+
+#neutron lb-healthmonitor-disassociate bf5eddd4-2e9d-4836-8d21-d9a8cec2162e c14f298c-f922-4743-9239-555a0670c7fb
+#...
diff --git a/04-lbaas-upgrade/migration_init/2-patch-lbaas-agent.sh b/04-lbaas-upgrade/migration_init/2-patch-lbaas-agent.sh
@@ -0,0 +1 @@
+# patch lbaas agent (bug 1600326)
diff --git a/04-lbaas-upgrade/migration_init/3-watch-haproxy.sh b/04-lbaas-upgrade/migration_init/3-watch-haproxy.sh
@@ -17,7 +17,7 @@ VERSION=""
 [ "$1" == "v2" ] && VERSION="v2/"
 
 #LBAAS_HOSTS=$(sudo mysql neutron -sNe 'select host from agents where `binary` = "neutron-lbaas-agent" and TIMESTAMPDIFF(SECOND, heartbeat_timestamp, NOw()) < 600' | awk 1 ORS=',')
-LBAAS_HOSTS=$(sudo mysql neutron -sNe 'select host from agents where `binary` = "neutron-lbaas-agent"' | awk 1 ORS=',')
+#LBAAS_HOSTS=$(sudo mysql neutron -sNe 'select host from agents where `binary` = "neutron-lbaas-agent"' | awk 1 ORS=',')
 
 while true; do
   if [ -z "$VERSION" ]; then
diff --git a/04-lbaas-upgrade/migration_steps/0-source-me.sh b/04-lbaas-upgrade/migration_steps/0-source-me.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export LBAAS_HOSTS=$(mysql neutron -sNe 'select host from agents where `binary` = "neutron-lbaas-agent" and TIMESTAMPDIFF(SECOND, heartbeat_timestamp, NOw()) < 300' | awk 1 ORS=',')
+export LBAAS_HOSTS_COUNT=$(mysql neutron -sNe 'select count(*) from agents where `binary` = "neutron-lbaas-agent" and TIMESTAMPDIFF(SECOND, heartbeat_timestamp, NOw()) < 300')
diff --git a/04-lbaas-upgrade/migration_steps/1-perform-alembic.sh b/04-lbaas-upgrade/migration_steps/1-perform-alembic.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+sudo cp ../4aff599b9845_lbaas_v1_to_v2_migration.py /usr/lib/python2.7/site-packages/neutron_lbaas/db/migration/alembic_migrations/versions/liberty/expand/
+sudo sed -i 's/3345facd0452/4aff599b9845/g' /usr/lib/python2.7/site-packages/neutron_lbaas/db/migration/alembic_migrations/versions/HEADS
+
+sudo neutron-db-manage --subproject neutron-lbaas upgrade head
+
+sudo rm /usr/lib/python2.7/site-packages/neutron_lbaas/db/migration/alembic_migrations/versions/liberty/expand/4aff599b9845_lbaas_v1_to_v2_migration.py
+sudo sed -i 's/4aff599b9845/3345facd0452/g' /usr/lib/python2.7/site-packages/neutron_lbaas/db/migration/alembic_migrations/versions/HEADS
+sudo mysql neutron -e "update alembic_version_lbaas set version_num = '3345facd0452' where version_num = '4aff599b9845'"
diff --git a/04-lbaas-upgrade/migration_steps/2-heat-neutron-db-backup.sh b/04-lbaas-upgrade/migration_steps/2-heat-neutron-db-backup.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+backup_dir=backup-$(hostname)
+mkdir $backup_dir
+
+sudo mysqldump --no-create-info --no-create-db --extended-insert --single-transaction --complete-insert --insert-ignore \
+        neutron members poolmonitorassociations poolstatisticss pools poolloadbalanceragentbindings vips healthmonitors | gzip > $backup_dir/neutron-lbaas-v1-pre-upgrade-backup.sql.gz
+
+sudo mysqldump --add-drop-database heat | gzip > $backup_dir/heat.sql.gz
+
diff --git a/04-lbaas-upgrade/migration_steps/3-switch-to-v2-on-agents.sh b/04-lbaas-upgrade/migration_steps/3-switch-to-v2-on-agents.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+test -z "$LBAAS_HOSTS" && echo "LBAAS_HOSTS env variable missing!" && exit
+
+ansible -i "$LBAAS_HOSTS" all -b -K -m script -a ansible-switch-to-v2-agent.sh
diff --git a/04-lbaas-upgrade/migration_steps/4-switch-to-v2-on-server.sh b/04-lbaas-upgrade/migration_steps/4-switch-to-v2-on-server.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+sudo cp -n /etc/neutron/neutron.conf /etc/neutron/neutron.conf.lbaasv1
+sudo cp -n /etc/neutron/neutron_lbaas.conf /etc/neutron/neutron_lbaas.conf.lbaasv1
+sudo cp -n /etc/neutron/lbaas_agent.ini /etc/neutron/lbaas_agent.ini.lbaasv1
+
+sudo crudini --set /etc/neutron/neutron.conf DEFAULT service_plugins router,neutron_lbaas.services.loadbalancer.plugin.LoadBalancerPluginv2
+sudo crudini --set /etc/neutron/neutron_lbaas.conf service_providers service_provider LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
+sudo crudini --set /etc/neutron/lbaas_agent.ini DEFAULT device_driver neutron_lbaas.drivers.haproxy.namespace_driver.HaproxyNSDriver
+
+sudo systemctl restart neutron-server.service
diff --git a/04-lbaas-upgrade/migration_steps/5-restart-lbaas-v2-agents.sh b/04-lbaas-upgrade/migration_steps/5-restart-lbaas-v2-agents.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+ansible -i "$LBAAS_HOSTS" all -b -K -m systemd -a "name=neutron-lbaasv2-agent state=restarted"
+
+echo "Waiting for $LBAAS_HOSTS_COUNT v2 agents to be registered..."
+LBAASV2_HOSTS_COUNT=0
+
+while [ "$LBAAS_HOSTS_COUNT" -ne "$LBAASV2_HOSTS_COUNT" ];
+do
+  sleep 3
+  LBAASV2_HOSTS_COUNT=$(mysql neutron -sNe 'select count(*) from agents where `binary` = "neutron-lbaasv2-agent"')
+  echo "Registered $LBAASV2_HOSTS_COUNT v2 agents so far..."
+done
diff --git a/04-lbaas-upgrade/migration_steps/6-lbaas-v2-db-changes.sh b/04-lbaas-upgrade/migration_steps/6-lbaas-v2-db-changes.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+sudo mysql neutron << 'EOF'
+# update ports device_owner
+update ports set device_owner = 'neutron:LOADBALANCERV2' where device_owner = 'neutron:LOADBALANCER';
+
+# update lbaas_loadbalanceragentbinding (lbaas agent v1 in use)
+update lbaas_loadbalanceragentbindings lbab inner join agents a on (lbab.agent_id = a.id) set lbab.agent_id = IFNULL((select id from agents where `binary` = 'neutron-lbaasv2-agent' and host = a.host), lbab.agent_id);
+EOF
diff --git a/04-lbaas-upgrade/migration_steps/7-restart-lbaas-v2-agents.sh b/04-lbaas-upgrade/migration_steps/7-restart-lbaas-v2-agents.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+ansible -i "$LBAAS_HOSTS" all -b -K -m systemd -a "name=neutron-lbaasv2-agent state=restarted"
+
+echo "Waiting for $LBAAS_HOSTS_COUNT v2 agents to be registered..."
+LBAASV2_HOSTS_COUNT=0
+
+while [ "$LBAAS_HOSTS_COUNT" -ne "$LBAASV2_HOSTS_COUNT" ];
+do
+  sleep 3
+  LBAASV2_HOSTS_COUNT=$(mysql neutron -sNe 'select count(*) from agents where `binary` = "neutron-lbaasv2-agent"')
+  echo "Registered $LBAASV2_HOSTS_COUNT v2 agents so far..."
+done
diff --git a/04-lbaas-upgrade/migration_steps/8-put-down-v1-haproxies.sh b/04-lbaas-upgrade/migration_steps/8-put-down-v1-haproxies.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+ansible -i "$LBAAS_HOSTS" all -b -K -m script -a ansible-signal-v1-haproxy.sh
diff --git a/04-lbaas-upgrade/migration_steps/9-translate-heat-stacks.sh b/04-lbaas-upgrade/migration_steps/9-translate-heat-stacks.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+sudo python ../lbaas_upgrade.py
+
+sudo mysql heat << 'EOF'
+update raw_template set files = REPLACE(files, 'OS::Neutron::PoolMember', 'OS::Neutron::LBaaS::PoolMember');
+update raw_template set files = REPLACE(files, 'pool_id', 'pool');
+EOF
+
+echo "Start heat api if everything went well!"
+echo "systemctl -M heat_newton start openstack-heat-api"
diff --git a/04-lbaas-upgrade/migration_steps/ansible-signal-v1-haproxy.sh b/04-lbaas-upgrade/migration_steps/ansible-signal-v1-haproxy.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+uuids=$(ls /var/lib/neutron/lbaas/)
+uuids=$(for uuid in $uuids; do ps aux | grep -q "/[v]ar/lib/neutron/lbaas/v2/$uuid" && echo $uuid; done)
+pids=$(for uuid in $uuids; do ps aux | grep "/[v]ar/lib/neutron/lbaas/$uuid" | tr -s " " | cut -d " " -f 2; done);
+for pid in $pids; do kill -s SIGTTOU -s SIGUSR1 $pid; echo "Sending SIGTTOU & SIGUSR1 to pid $pid"; done
diff --git a/04-lbaas-upgrade/migration_steps/ansible-signal-v2-haproxy.sh b/04-lbaas-upgrade/migration_steps/ansible-signal-v2-haproxy.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+uuids=$(ls /var/lib/neutron/lbaas/v2/)
+uuids=$(for uuid in $uuids; do ps aux | grep -q "/[v]ar/lib/neutron/lbaas/$uuid" && echo $uuid; done)
+pids=$(for uuid in $uuids; do ps aux | grep "/[v]ar/lib/neutron/lbaas/v2/$uuid" | tr -s " " | cut -d " " -f 2; done);
+for pid in $pids; do kill -s SIGTTOU -s SIGUSR1 $pid; echo "Sending SIGTTOU & SIGUSR1 to pid $pid"; done
diff --git a/04-lbaas-upgrade/migration_steps/ansible-switch-to-v1-agent.sh b/04-lbaas-upgrade/migration_steps/ansible-switch-to-v1-agent.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+sudo cp -f /etc/neutron/neutron.conf.lbaasv1 /etc/neutron/neutron.conf 
+sudo cp -f /etc/neutron/neutron_lbaas.conf.lbaasv1 /etc/neutron/neutron_lbaas.conf 
+sudo cp -f /etc/neutron/lbaas_agent.ini.lbaasv1 /etc/neutron/lbaas_agent.ini 
+
+sudo systemctl disable neutron-lbaasv2-agent
+sudo systemctl stop neutron-lbaasv2-agent
+sudo systemctl enable neutron-lbaas-agent
diff --git a/04-lbaas-upgrade/migration_steps/ansible-switch-to-v2-agent.sh b/04-lbaas-upgrade/migration_steps/ansible-switch-to-v2-agent.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+sudo cp -n /etc/neutron/neutron.conf /etc/neutron/neutron.conf.lbaasv1
+sudo cp -n /etc/neutron/neutron_lbaas.conf /etc/neutron/neutron_lbaas.conf.lbaasv1
+sudo cp -n /etc/neutron/lbaas_agent.ini /etc/neutron/lbaas_agent.ini.lbaasv1
+
+sudo crudini --set /etc/neutron/neutron.conf DEFAULT service_plugins router,neutron_lbaas.services.loadbalancer.plugin.LoadBalancerPluginv2
+sudo crudini --set /etc/neutron/neutron_lbaas.conf service_providers service_provider LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
+sudo crudini --set /etc/neutron/lbaas_agent.ini DEFAULT device_driver neutron_lbaas.drivers.haproxy.namespace_driver.HaproxyNSDriver
+
+sudo systemctl disable neutron-lbaas-agent
+sudo systemctl stop neutron-lbaas-agent
+sudo systemctl enable neutron-lbaasv2-agent
diff --git a/04-lbaas-upgrade/migration_steps/r0-import-db-backup.sh b/04-lbaas-upgrade/migration_steps/r0-import-db-backup.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+backup_dir=backup-$(hostname)
+cd $backup_dir
+
+test -f heat.sql.gz && gunzip heat.sql.gz
+mysql heat < heat.sql
+cd -
+
diff --git a/04-lbaas-upgrade/migration_steps/r1-switch-to-v1-on-agents.sh b/04-lbaas-upgrade/migration_steps/r1-switch-to-v1-on-agents.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+test -z "$LBAAS_HOSTS" && echo "LBAAS_HOSTS env variable missing!" && exit
+
+ansible -i "$LBAAS_HOSTS" all -b -K -m script -a ansible-switch-to-v1-agent.sh
diff --git a/04-lbaas-upgrade/migration_steps/r2-switch-to-v1-on-server.sh b/04-lbaas-upgrade/migration_steps/r2-switch-to-v1-on-server.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+sudo cp -n /etc/neutron/neutron.conf.lbaasv1 /etc/neutron/neutron.conf
+sudo cp -n /etc/neutron/neutron_lbaas.conf.lbaasv1 /etc/neutron/neutron_lbaas.conf
+sudo cp -n /etc/neutron/lbaas_agent.ini.lbaasv1 /etc/neutron/lbaas_agent.ini
+
+sudo systemctl restart neutron-server.service
diff --git a/04-lbaas-upgrade/migration_steps/r3-lbaas-v1-db-changes.sh b/04-lbaas-upgrade/migration_steps/r3-lbaas-v1-db-changes.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+sudo mysql neutron << 'EOF'
+# update ports device_owner
+update ports set device_owner = 'neutron:LOADBALANCER' where device_owner = 'neutron:LOADBALANCERV2';
+
+# update lbaas_loadbalanceragentbinding (lbaas agent v2 in use)
+update lbaas_loadbalanceragentbindings lbab inner join agents a on (lbab.agent_id = a.id) set lbab.agent_id = IFNULL((select id from agents where `binary` = 'neutron-lbaas-agent' and host = a.host), lbab.agent_id);
+EOF
diff --git a/04-lbaas-upgrade/migration_steps/r4-restart-lbaas-v1-agents.sh b/04-lbaas-upgrade/migration_steps/r4-restart-lbaas-v1-agents.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+ansible -i "$LBAAS_HOSTS" all -b -K -m systemd -a "name=neutron-lbaas-agent state=restarted"
+
diff --git a/04-lbaas-upgrade/migration_steps/r5-put-down-v2-haproxies.sh b/04-lbaas-upgrade/migration_steps/r5-put-down-v2-haproxies.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+ansible -i "$LBAAS_HOSTS" all -b -K -m script -a ansible-signal-v2-haproxy.sh

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +# "clean' pools so that they have at most 1 health monitor
++
 +#neutron lb-healthmonitor-disassociate bf5eddd4-2e9d-4836-8d21-d9a8cec2162e c14f298c-f922-4743-9239-555a0670c7fb
 +#...