List Amphora VMs
openstack server list --project service
Fix broken/out Amphora VMs / Ports
http://eavesdrop.openstack.org/irclogs/%23openstack-lbaas/%23openstack-lbaas.2019-05-17.log.html
Fix broken amphora port DB_PASS=$(grep octavia_database_password /etc/kolla/passwords.yml | cut -d " " -f2) # check VRRP ports for i in $(openstack port list -f value | grep octavia-lb-vrrp| awk '{ print $1}'); do echo $i; openstack port show $i -c allowed_address_pairs -f value done > /tmp/vrrp.txt # List broken loadbalancer openstack loadbalancer list | grep -v ACTIVE openstack loadbalancer amphora list | grep -v ALLOCATED # Set LB ID LB_ID=b98ea4bb-0631-4be9-a9e8-8841bf56bf2f # openstack loadbalancer show ${LB_ID} # Show amphora status openstack loadbalancer amphora list --loadbalancer ${LB_ID} # ensure LB provisioning_status is ERROR (deprected?) # mysql --host=db.service.i.example.com --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \ # --execute="update load_balancer set provisioning_status = 'ERROR' where id = '${LB_ID}'" # OPTIONAL: delete standalone amphora entries # mysql --host=db.service.i.example.com --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="delete from amphora where role = 'STANDALONE' and load_balancer_id='${LB_ID}'" LB_HA_IP=$(mysql --host=db.service.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} --skip-column-names -s \ --execute="select distinct ha_ip from amphora where load_balancer_id='${LB_ID}'") # grep -B1 ${LB_HA_IP} /tmp/vrrp.txt # Check / show existing LB ports grep -B1 ${LB_HA_IP} /tmp/vrrp.txt | egrep '^[a-Z0-9]+' | \ xargs -L1 openstack port show -c id -f value # Show loadbalancer status in DB mysql --host=db.service.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="select * from amphora where load_balancer_id='${LB_ID}'" # check if both VRRP ports exist and got a MASTER and a BACKUP entry in the DB # "recover" a DELETED master or backup amphorae and ensure to use a unallocated fake IP for lb_network_ip LB_AMPHORA_ID=3db411b1-4b20-429f-80a5-3cb7c451876b mysql --host=db.service.i.example.com --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="update amphora set status='ALLOCATED', lb_network_ip='172.16.200.123' where id='${LB_AMPHORA_ID}'"; # OPTIONAL: delete broken amphora entry # mysql --host=db.service.i.example.com --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="delete from amphora where id = '${LB_AMPHORA_ID}' and load_balancer_id='${LB_ID}'" # ensure LB provisioning_status is ACTIVE mysql --host=db.service.i.example.com --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="update load_balancer set provisioning_status = 'ACTIVE' where id = '${LB_ID}'" # failover amphora openstack loadbalancer amphora list --loadbalancer ${LB_ID} openstack loadbalancer amphora failover ${LB_AMPHORA_ID} # DEBUG watch openstack loadbalancer amphora list --loadbalancer ${LB_ID}
Recreate VRRP port
# Show loadbalancer status in DB mysql --host=db.service.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="select * from amphora where load_balancer_id='${LB_ID}'" openstack port show 7ba30f9b-6506-4976-8f6f-9548fee42c6e openstack port create octavia-lb-vrrp-$(uuidgen) \ --network ${LB_NETWORK} \ --fixed-ip ip-address='${AMPHORA_IP}',subnet='${LB_SUBNET}' \ --security-group ${LB_SECURITY_GROUP} \ --allowed-address ip-address=${LB_HA_IP} mysql --host=db.service.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="update amphora set status='ALLOCATED', role='MASTER', vrrp_interface='eth1', vrrp_priority=100, vrrp_port_id='34aef610-d38f-4eca-8c7d-aad19e6e6f5f', vrrp_ip='192.168.248.32' where id='ecdcceeb-39a0-4a51-a872-6af81edc9ab2'";
Failover all Amphora VMs in status ERROR
LB_IDS=$(openstack loadbalancer amphora list --status ERROR -c loadbalancer_id -f value | sort -u) for LB_ID in ${LB_IDS}; do openstack loadbalancer failover ${LB_ID} sleep 300 openstack loadbalancer amphora list --loadbalancer ${LB_ID} done
Find broken Octavia Loadbalancer / Amphora VMs
The "loadbalancer amphora failover" command are avaiable since OSC Stein release: https://docs.openstack.org/releasenotes/python-octaviaclient/stein.html but the python-octaviaclient 1.7 OSC also works with OS Rocky realase. (pip install -U "python-octaviaclient==1.7.0")
DB_PASS=$(grep octavia_database_password /etc/kolla/passwords.yml | cut -d " " -f2) LB_IDS=$(openstack loadbalancer amphora list --status ERROR -c loadbalancer_id -f value | sort | uniq) for LB_ID in ${LB_IDS}; do # update LB status to ACTIVE mysql --host=db.service.stage.i.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="update load_balancer set provisioning_status='ACTIVE' where id='${LB_ID}';" # get amphora ID AMPHORA_IDS=$(openstack loadbalancer amphora list --status ERROR --loadbalancer ${LB_ID} -c id -f value) for AMPHORA_ID in ${AMPHORA_IDS}; do # failover broken amphora VMs openstack loadbalancer amphora failover ${AMPHORA_ID} sleep 60 done done
Manual debug Amphora VMs
# List broken LoadBalancer instances openstack loadbalancer list --provisioning-status ERROR openstack loadbalancer list --provisioning-status PENDING_UPDATE # List broken Loadbalancer VMs openstack loadbalancer amphora list --provisioning-status ERROR openstack loadbalancer amphora list --role STANDALONE
Show Loadbalancer state
LB_ID=0ce30f0e-1d75-486c-a09f-79125abf44b8 # List LoadBalancer details openstack loadbalancer show ${LB_ID} # List LoadBalancer VMs details openstack loadbalancer amphora list --loadbalancer ${LB_ID} # show project PROJECT_ID=$(openstack loadbalancer show -c project_id -f value ${LB_ID}) openstack project show -c name -f value ${PROJECT_ID} # show domain DOMAIN_ID=$(openstack project show -c domain_id -f value ${PROJECT_ID}) openstack domain show -c name -f value ${DOMAIN_ID}
Manual update provisioning_status from PENDING_UPDATE / ERROR state to ACTIVE in Octavia Database
Currently there is no OpenStack CLI command to change the provisioning_status with the CLI
DB_PASS=$(grep octavia_database_password /etc/kolla/passwords.yml | cut -d " " -f2) mysql --host=db.service.example.com --port=3306 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="update load_balancer set provisioning_status = 'ACTIVE' where id = '${LB_ID}'"
Trigger manual LoadBalancer failover
This will cause a short interruption of the LB service since both VMs will by recreated
# manually failover loadbalancer openstack loadbalancer failover ${LB_ID} # manually failover amphora vm AMPHORA_ID=$(openstack loadbalancer amphora list --loadbalancer ${LB_ID} --status ERROR -c id -f value) openstack loadbalancer amphora failover ${AMPHORA_ID}
Check LoadBalancer state
# check LoadBalancer state openstack loadbalancer show ${LB_ID} # check LoadBalancer VMs state openstack loadbalancer amphora list --loadbalancer ${LB_ID}
Debug
# list all loadbalancer instances openstack loadbalancer list # list all Amphora instances openstack loadbalancer amphora list # List all Amphora VMs openstack server list --all --long --name amphora # get LoadBalancer count openstack loadbalancer list -c id -f value | wc -l # get Amphora VMs count openstack loadbalancer amphora list -c loadbalancer_id -f value | wc -l openstack server list --all --long --name amphora -c Name -f value | wc -l # restart LB VM openstack server reboot --soft amphora-${AMPHORA_ID} # show LB VM console openstack console log show amphora-${AMPHORA_ID} # check amphora vs. server VM openstack server list --all-projects --name amphora -c Name -f value | cut -d "-" -f2- | sort > /tmp/amphora_vms openstack loadbalancer amphora list --status ALLOCATED -c id -f value | sort > /tmp/amphora_lb diff /tmp/amphora_vms /tmp/amphora_lb openstack loadbalancer amphora list | grep 558196ab-6215-4f4d-a4e8-9eee2a300d33 openstack server show amphora-558196ab-6215-4f4d-a4e8-9eee2a300d33 openstack loadbalancer amphora failover 558196ab-6215-4f4d-a4e8-9eee2a300d33
Debug DB
# show amphora VMs openstack loadbalancer amphora list --loadbalancer ${LB_ID} # show defined LB MAC / IP LB_VIP_NETWORK_ID=$(openstack loadbalancer show ${LB_ID} -c vip_network_id -f value) openstack port list --network ${LB_VIP_NETWORK_ID} | grep vrrp LB_PORT_IDS=$(openstack port list --network ${LB_VIP_NETWORK_ID} | grep vrrp | cut -d" " -f2) for LB_PORT_ID in ${LB_PORT_IDS}; do LB_PORT_DEVICE_ID=$(openstack port show ${LB_PORT_ID} -c device_id -f value) openstack server show ${LB_PORT_DEVICE_ID} -c name -f value done # show used VM MAC / IP LB_AMPHORA_VMS=$(openstack loadbalancer amphora list --loadbalancer ${LB_ID} -c id -f value) for LB_AMPHORA_VM in ${LB_AMPHORA_VMS}; do openstack port list --server amphora-${LB_AMPHORA_VM} done # show definded MAC / IP in DB mysql --host=db.service.i.example.net --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="select * from amphora where load_balancer_id='${LB_ID}' order by updated_at desc" LB_AMPHORA_ID=606cf400-b626-4473-8343-5d25b1c04a0d # update amphora status ERROR to DELETED mysql --host=db.service.i.example.net --port=6033 --database=octavia --user=octavia --password=${DB_PASS} \ --execute="update amphora set status='DELETED' where id='${LB_AMPHORA_ID}'"; openstack server delete amphora-${LB_AMPHORA_ID} mysql --host=db.service.i.example.net --port=6033 --database=octavia --user=octavia --password=${DB_PASS} --execute="update amphora set status='ALLOCATED', role='BACKUP', vrrp_priority=90, vrrp_port_id='f53eb837-cf88-4008-b019-5a6637e33d48', vrrp_ip='10.15.10.19' where id='45d90b49-45af-4c87-9f72-33929178e929'"; openstack loadbalancer list | grep -v ACTIVE openstack loadbalancer amphora list | grep -Ev "ALLOCATED|READY" openstack loadbalancer amphora list | grep -Ev "MASTER|BACKUP|READY" # check VRRP ports for i in $(openstack port list -f value | grep octavia-lb-vrrp| awk '{ print $1}'); do echo $i; openstack port show $i -c allowed_address_pairs -f value done > /tmp/vrrp.txt LB_HA_IP=$(openstack loadbalancer show ${LB_ID} -c vip_address -f value) grep -B1 ${LB_HA_IP} /tmp/vrrp.txt