From 526a86f2857df355b03514c02383426bc29a53ae Mon Sep 17 00:00:00 2001 From: Roman Safronov Date: Thu, 21 Mar 2024 00:07:59 +0200 Subject: [PATCH] Add functions for power operations This patch adds functions for performing some power operations on nodes of a podified openstack environment. It's assumed that the podified environment is a virtual testing environment deployed by ci-framework [1]. On such environment all openstack nodes are virtual machines running on a hypervisor which is accessible by password-less ssh from a deployment host (aka ansible host, also referred as a proxy host in the code). Also, applied the functions in the relevant internal DNS test. [1] https://github.com/openstack-k8s-operators/ci-framework Change-Id: Ica919fffb770c3c17a6cfd023fe36ef71544ba63 --- whitebox_neutron_tempest_plugin/config.py | 15 +++-- .../tests/scenario/base.py | 63 +++++++++++++++++++ .../tests/scenario/test_internal_dns.py | 36 +++++------ 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/whitebox_neutron_tempest_plugin/config.py b/whitebox_neutron_tempest_plugin/config.py index c0c5a39..dca131d 100644 --- a/whitebox_neutron_tempest_plugin/config.py +++ b/whitebox_neutron_tempest_plugin/config.py @@ -38,10 +38,6 @@ WhiteboxNeutronPluginOptions = [ cfg.StrOpt('tester_key_file', default='', help='Key file to access host to execute validated commands.'), - cfg.BoolOpt('node_power_change', - default=True, - help='Whether to power off/on nodes, ' - 'such as controller/compute.'), cfg.StrOpt('openstack_type', default='devstack', help='Type of openstack deployment, ' @@ -85,6 +81,12 @@ WhiteboxNeutronPluginOptions = [ default=False, help='Boolean that specifies if Provider Routed Networks' 'are supported or not'), + cfg.BoolOpt('run_power_operations_tests', + default=False, + help='Specify explicitly whether to run tests that perform ' + 'power operations, like shutdown/startup openstack nodes.' + 'These tests can be disruptive and not suitable for some ' + 'environments.'), cfg.IntOpt('broadcast_receivers_count', default=2, help='How many receivers to use in broadcast tests. Default ' @@ -150,6 +152,11 @@ WhiteboxNeutronPluginOptions = [ 'when this time expires. This is needed in order to stop ' 'remote process in case test or connection was ' 'interrupted unexpectedly.'), + cfg.StrOpt('hypervisor_host', + default='hypervisor-1', + help='Hypervisor host for podified environment based on libvirt' + 'virtual machines, typically deployed by ci-framework: ' + 'https://github.com/openstack-k8s-operators/ci-framework'), cfg.StrOpt('proxy_host_address', default='', help='Intermediate host to run commands on podified ' diff --git a/whitebox_neutron_tempest_plugin/tests/scenario/base.py b/whitebox_neutron_tempest_plugin/tests/scenario/base.py index 042a568..a4713fa 100644 --- a/whitebox_neutron_tempest_plugin/tests/scenario/base.py +++ b/whitebox_neutron_tempest_plugin/tests/scenario/base.py @@ -698,6 +698,69 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase): 'Command failure "{}" on "{}" nodes.'.format( cmd, group_name)) + def find_host_virsh_name(cls, host): + cmd = ("timeout 10 ssh {} sudo virsh list --name | grep -w {}").format( + WB_CONF.hypervisor_host, host) + return cls.proxy_host_client.exec_command(cmd).strip() + + @classmethod + def is_host_state_is_shut_off(cls, host): + cmd = ("timeout 10 ssh {} virsh list --state-shutoff | grep -w {} " + "|| true".format(WB_CONF.hypervisor_host, host)) + output = cls.proxy_host_client.exec_command(cmd) + return True if host in output else False + + @classmethod + def is_host_loginable(cls, host): + cmd = "timeout 10 ssh {} ssh {} hostname || true".format( + WB_CONF.hypervisor_host, host) + output = cls.proxy_host_client.exec_command(cmd) + return True if host in output else False + + @classmethod + def power_off_host(cls, host): + if not WB_CONF.run_power_operations_tests: + raise cls.skipException("Power operations are not allowed") + cmd = "timeout 10 ssh {} sudo virsh destroy {}".format( + WB_CONF.hypervisor_host, cls.find_host_virsh_name()) + cls.proxy_host_client.exec_command(cmd) + common_utils.wait_until_true( + lambda: cls.is_host_state_is_shut_off(host), + timeout=30, sleep=5) + + @classmethod + def power_on_host(cls, host): + if not WB_CONF.run_power_operations_tests: + raise cls.skipException("Power operations are not allowed") + cmd = "timeout 10 ssh {} sudo virsh start {}".format( + WB_CONF.hypervisor_host, cls.find_host_virsh_name()) + cls.proxy_host_client.exec_command(cmd) + # TODO(rsafrono): implement and apply additional health checks + common_utils.wait_until_true( + lambda: cls.is_host_loginable(host), + timeout=120, sleep=5) + + @classmethod + def reboot_host(cls, host): + if not WB_CONF.run_power_operations_tests: + raise cls.skipException("Power operations are not allowed") + cmd = "timeout 10 ssh {} sudo virsh reboot {}".format( + WB_CONF.hypervisor_host, cls.find_host_virsh_name()) + cls.proxy_host_client.exec_command(cmd) + common_utils.wait_until_true( + lambda: cls.is_host_loginable(host), + timeout=120, sleep=5) + + def ensure_overcloud_nodes_active(self): + """Checks all openstack nodes are up, otherwise activates them. + """ + # get overcloud nodes info if it doesn't exist + if not hasattr(self, 'nodes'): + self.discover_nodes() + for node in self.nodes: + if self.is_host_state_is_shut_off(node['name']): + self.power_on_host(node['name']) + class BaseTempestTestCaseAdvanced(BaseTempestWhiteboxTestCase): """Base class skips test suites unless advanced image is available, diff --git a/whitebox_neutron_tempest_plugin/tests/scenario/test_internal_dns.py b/whitebox_neutron_tempest_plugin/tests/scenario/test_internal_dns.py index cad2a84..bfe8ef4 100644 --- a/whitebox_neutron_tempest_plugin/tests/scenario/test_internal_dns.py +++ b/whitebox_neutron_tempest_plugin/tests/scenario/test_internal_dns.py @@ -23,8 +23,6 @@ from tempest.common import utils from tempest.lib.common.utils import data_utils from tempest.lib import decorators from tempest.lib.exceptions import SSHExecCommandFailed -# TODO(mblue): implement alternative for next gen -# from tempest_helper_plugin.common.utils.linux import node_power from whitebox_neutron_tempest_plugin.tests.scenario import base @@ -300,12 +298,22 @@ class InternalDNSInterruptionsAdvancedTestOvn( def skip_checks(cls): super(InternalDNSInterruptionsAdvancedTestOvn, cls).skip_checks() if WB_CONF.openstack_type == 'devstack': - cls.skipException( + raise cls.skipException( "Devstack doesn't support powering nodes on/off, " "skipping tests") - if not WB_CONF.node_power_change: - cls.skipException( - "node_power_change is not enabled, skipping tests") + if not WB_CONF.run_power_operations_tests: + raise cls.skipException( + "run_power_operations_tests config is not enabled, " + "skipping tests") + + @classmethod + def resource_setup(cls): + super(InternalDNSInterruptionsAdvancedTestOvn, cls).resource_setup() + for node in cls.nodes: + if node['is_networker'] is True and node['is_compute'] is True: + raise cls.skipException( + "Not supported when environment allows OVN gateways on " + "compute nodes.") @decorators.attr(type='slow') @utils.requires_ext(extension="dns-integration", service="network") @@ -329,10 +337,8 @@ class InternalDNSInterruptionsAdvancedTestOvn( # when a VM is created, it is a known limitation. # Therefore VM's dns-name/hostname is checked to be as VM's name. - # TODO(mblue): implement alternative for next gen # ensures overcloud nodes are up for next tests - self.skipException("Powering nodes not supported yet (TODO)") - # self.addCleanup(self.ensure_overcloud_nodes_active) + self.addCleanup(self.ensure_overcloud_nodes_active) # create port with dns-name (as VM name) vm_name = self._rand_name('vm') dns_port = self.create_port(self.network, @@ -347,19 +353,11 @@ class InternalDNSInterruptionsAdvancedTestOvn( vm_1['ssh_client'] = self._create_ssh_client( vm_1['fip']['floating_ip_address']) self._get_router_and_nodes_info() - # TODO(mblue): implement alternative for next gen - # node_id = self.hosts_info.get_overcloud_node_id( - # self.router_gateway_chassis) # soft shutdown master networker node - # TODO(mblue): implement alternative for next gen - # node_power.power_off(node_id) + self.power_off_host(self.router_gateway_chassis) # validate hostname (dns-name) using API, guest VM, # and OVN NBDB when networker node is off and on self._dns_all_validations(vm_name, dns_port, vm_1['ssh_client']) # turn on networker node, wait until it is up and working - # TODO(mblue): implement alternative for next gen - # node_power.power_on( - # node_id, - # services_clients=[self.os_admin.network_client], - # agents_clients=[self.os_admin.compute.ServicesClient()]) + self.power_on_host(self.router_gateway_chassis) self._dns_all_validations(vm_name, dns_port, vm_1['ssh_client'])