Add octavia health checks to faults/ha tests

The disruptions executed during these tests may affect any existing
resources, previously created on the cloud. The current health checks
validate status of VMs, pacemaker, services, etc. This patch adds health
checks for any existing LBs.

Change-Id: I13563d639c40fb2d454a8aa1bf6f5c42ff72de78

tobiko/openstack/octavia/__init__.py

@@ -27,6 +27,7 @@ get_amphora_compute_node = _amphora.get_amphora_compute_node
 get_master_amphora = _amphora.get_master_amphora
 list_amphorae = _amphora.list_amphorae
 get_amphora_stats = _amphora.get_amphora_stats
+get_amphora = _amphora.get_amphora
 run_command_on_amphora = _amphora.run_command_on_amphora
 
 OCTAVIA_CLIENT_CLASSSES = _client.OCTAVIA_CLIENT_CLASSSES
@@ -35,6 +36,7 @@ octavia_client = _client.octavia_client
 OctaviaClientFixture = _client.OctaviaClientFixture
 OctaviaClientType = _client.OctaviaClientType
 list_members = _client.list_members
+list_load_balancers = _client.list_load_balancers
 find_load_balancer = _client.find_load_balancer
 create_load_balancer = _client.create_load_balancer
 find_listener = _client.find_listener
@@ -61,9 +63,11 @@ AmphoraMgmtPortNotFound = _exceptions.AmphoraMgmtPortNotFound
 
 # Constants
 PROVISIONING_STATUS = _constants.PROVISIONING_STATUS
+OPERATING_STATUS = _constants.OPERATING_STATUS
 ACTIVE = _constants.ACTIVE
 ERROR = _constants.ERROR
 PENDING_UPDATE = _constants.PENDING_UPDATE
+ONLINE = _constants.ONLINE
 PROTOCOL_HTTP = _constants.PROTOCOL_HTTP
 PROTOCOL_TCP = _constants.PROTOCOL_TCP
 LB_ALGORITHM_ROUND_ROBIN = _constants.LB_ALGORITHM_ROUND_ROBIN

tobiko/openstack/octavia/_client.py

@@ -81,6 +81,11 @@ def list_members(pool_id: str):
     return os_sdk_client.load_balancer.members(pool=pool_id)
 
 
+def list_load_balancers(**lb_kwargs):
+    os_sdk_client = openstacksdkclient.openstacksdk_client()
+    return os_sdk_client.load_balancer.load_balancers(**lb_kwargs)
+
+
 def find_load_balancer(lb_name: str):
     os_sdk_client = openstacksdkclient.openstacksdk_client()
     return os_sdk_client.load_balancer.find_load_balancer(lb_name)

tobiko/openstack/octavia/_constants.py

@@ -15,11 +15,13 @@
 
 # Octavia attributes
 PROVISIONING_STATUS = 'provisioning_status'
+OPERATING_STATUS = 'operating_status'
 
-# Octavia provisioning status
+# Octavia provisioning and operating status
 ACTIVE = 'ACTIVE'
 ERROR = 'ERROR'
 PENDING_UPDATE = 'PENDING_UPDATE'
+ONLINE = 'ONLINE'
 
 # Octavia protocols
 PROTOCOL_HTTP = 'HTTP'

tobiko/tests/faults/ha/test_cloud_recovery.py

@@ -23,8 +23,10 @@ import testtools
 
 import tobiko
 from tobiko import config
+from tobiko.openstack import keystone
 from tobiko.openstack import neutron
 from tobiko.openstack import nova as nova_osp
+from tobiko.openstack import octavia
 from tobiko.openstack import topology
 from tobiko.openstack import tests
 from tobiko.tests.faults.ha import cloud_disruptions
@@ -66,6 +68,8 @@ def overcloud_health_checks(passive_checks_only=False,
     if not skip_mac_table_size_test:
         tests.test_ovs_bridges_mac_table_size()
 
+    octavia_health_checks()
+
 
 # check vm create with ssh and ping checks
 def check_vm_create():
@@ -92,6 +96,23 @@ def check_overcloud_processes_health():
             procs.ovn_overcloud_processes_validations)
 
 
+def octavia_health_checks():
+    if keystone.is_service_missing(name='octavia'):
+        LOG.debug('octavia service not available')
+        return
+    # check LBs and Amphorae are healthy
+    LOG.debug("check all LBs are in healthy status")
+    for lb in octavia.list_load_balancers():
+        LOG.debug("checkin LBs:\n%s", lb)
+        octavia.wait_for_status(lb['id'], interval=5., timeout=120.)
+        octavia.wait_for_status(lb['id'],
+                                status_key=octavia.OPERATING_STATUS,
+                                status=octavia.ONLINE,
+                                interval=5.,
+                                timeout=120.)
+    LOG.debug("All LBs are in healthy status")
+
+
 class OvercloudHealthCheck(tobiko.SharedFixture):
 
     skips: typing.FrozenSet[str] = frozenset()