Add functions for power operations

This patch adds functions for performing some power operations
on nodes of a podified openstack environment.
It's assumed that the podified environment is a virtual testing
environment deployed by ci-framework [1].
On such environment all openstack nodes are virtual machines running
on a hypervisor which is accessible by password-less ssh from
a deployment host (aka ansible host, also referred as a proxy host
in the code).
Also, applied the functions in the relevant internal DNS test.

[1] https://github.com/openstack-k8s-operators/ci-framework

Change-Id: Ica919fffb770c3c17a6cfd023fe36ef71544ba63

whitebox_neutron_tempest_plugin/config.py

@@ -38,10 +38,6 @@ WhiteboxNeutronPluginOptions = [
     cfg.StrOpt('tester_key_file',
                default='',
                help='Key file to access host to execute validated commands.'),
-    cfg.BoolOpt('node_power_change',
-                default=True,
-                help='Whether to power off/on nodes, '
-                'such as controller/compute.'),
     cfg.StrOpt('openstack_type',
                default='devstack',
                help='Type of openstack deployment, '
@@ -85,6 +81,12 @@ WhiteboxNeutronPluginOptions = [
                 default=False,
                 help='Boolean that specifies if Provider Routed Networks'
                      'are supported or not'),
+    cfg.BoolOpt('run_power_operations_tests',
+                default=False,
+                help='Specify explicitly whether to run tests that perform '
+                     'power operations, like shutdown/startup openstack nodes.'
+                     'These tests can be disruptive and not suitable for some '
+                     'environments.'),
     cfg.IntOpt('broadcast_receivers_count',
                default=2,
                help='How many receivers to use in broadcast tests. Default '
@@ -150,6 +152,11 @@ WhiteboxNeutronPluginOptions = [
                     'when this time expires. This is needed in order to stop '
                     'remote process in case test or connection was '
                     'interrupted unexpectedly.'),
+    cfg.StrOpt('hypervisor_host',
+               default='hypervisor-1',
+               help='Hypervisor host for podified environment based on libvirt'
+                    'virtual machines, typically deployed by ci-framework: '
+                    'https://github.com/openstack-k8s-operators/ci-framework'),
     cfg.StrOpt('proxy_host_address',
                default='',
                help='Intermediate host to run commands on podified '

whitebox_neutron_tempest_plugin/tests/scenario/base.py

@@ -698,6 +698,69 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
                         'Command failure "{}" on "{}" nodes.'.format(
                             cmd, group_name))
 
+    def find_host_virsh_name(cls, host):
+        cmd = ("timeout 10 ssh {} sudo virsh list --name | grep -w {}").format(
+            WB_CONF.hypervisor_host, host)
+        return cls.proxy_host_client.exec_command(cmd).strip()
+
+    @classmethod
+    def is_host_state_is_shut_off(cls, host):
+        cmd = ("timeout 10 ssh {} virsh list --state-shutoff | grep -w {} "
+               "|| true".format(WB_CONF.hypervisor_host, host))
+        output = cls.proxy_host_client.exec_command(cmd)
+        return True if host in output else False
+
+    @classmethod
+    def is_host_loginable(cls, host):
+        cmd = "timeout 10 ssh {} ssh {} hostname || true".format(
+            WB_CONF.hypervisor_host, host)
+        output = cls.proxy_host_client.exec_command(cmd)
+        return True if host in output else False
+
+    @classmethod
+    def power_off_host(cls, host):
+        if not WB_CONF.run_power_operations_tests:
+            raise cls.skipException("Power operations are not allowed")
+        cmd = "timeout 10 ssh {} sudo virsh destroy {}".format(
+            WB_CONF.hypervisor_host, cls.find_host_virsh_name())
+        cls.proxy_host_client.exec_command(cmd)
+        common_utils.wait_until_true(
+            lambda: cls.is_host_state_is_shut_off(host),
+            timeout=30, sleep=5)
+
+    @classmethod
+    def power_on_host(cls, host):
+        if not WB_CONF.run_power_operations_tests:
+            raise cls.skipException("Power operations are not allowed")
+        cmd = "timeout 10 ssh {} sudo virsh start {}".format(
+            WB_CONF.hypervisor_host, cls.find_host_virsh_name())
+        cls.proxy_host_client.exec_command(cmd)
+        # TODO(rsafrono): implement and apply additional health checks
+        common_utils.wait_until_true(
+            lambda: cls.is_host_loginable(host),
+            timeout=120, sleep=5)
+
+    @classmethod
+    def reboot_host(cls, host):
+        if not WB_CONF.run_power_operations_tests:
+            raise cls.skipException("Power operations are not allowed")
+        cmd = "timeout 10 ssh {} sudo virsh reboot {}".format(
+            WB_CONF.hypervisor_host, cls.find_host_virsh_name())
+        cls.proxy_host_client.exec_command(cmd)
+        common_utils.wait_until_true(
+            lambda: cls.is_host_loginable(host),
+            timeout=120, sleep=5)
+
+    def ensure_overcloud_nodes_active(self):
+        """Checks all openstack nodes are up, otherwise activates them.
+        """
+        # get overcloud nodes info if it doesn't exist
+        if not hasattr(self, 'nodes'):
+            self.discover_nodes()
+        for node in self.nodes:
+            if self.is_host_state_is_shut_off(node['name']):
+                self.power_on_host(node['name'])
+
 
 class BaseTempestTestCaseAdvanced(BaseTempestWhiteboxTestCase):
     """Base class skips test suites unless advanced image is available,

whitebox_neutron_tempest_plugin/tests/scenario/test_internal_dns.py

@@ -23,8 +23,6 @@ from tempest.common import utils
 from tempest.lib.common.utils import data_utils
 from tempest.lib import decorators
 from tempest.lib.exceptions import SSHExecCommandFailed
-# TODO(mblue): implement alternative for next gen
-# from tempest_helper_plugin.common.utils.linux import node_power
 
 from whitebox_neutron_tempest_plugin.tests.scenario import base
 
@@ -300,12 +298,22 @@ class InternalDNSInterruptionsAdvancedTestOvn(
     def skip_checks(cls):
         super(InternalDNSInterruptionsAdvancedTestOvn, cls).skip_checks()
         if WB_CONF.openstack_type == 'devstack':
-            cls.skipException(
+            raise cls.skipException(
                 "Devstack doesn't support powering nodes on/off, "
                 "skipping tests")
-        if not WB_CONF.node_power_change:
+        if not WB_CONF.run_power_operations_tests:
-            cls.skipException(
+            raise cls.skipException(
-                "node_power_change is not enabled, skipping tests")
+                "run_power_operations_tests config is not enabled, "
+                "skipping tests")
+
+    @classmethod
+    def resource_setup(cls):
+        super(InternalDNSInterruptionsAdvancedTestOvn, cls).resource_setup()
+        for node in cls.nodes:
+            if node['is_networker'] is True and node['is_compute'] is True:
+                raise cls.skipException(
+                    "Not supported when environment allows OVN gateways on "
+                    "compute nodes.")
 
     @decorators.attr(type='slow')
     @utils.requires_ext(extension="dns-integration", service="network")
@@ -329,10 +337,8 @@ class InternalDNSInterruptionsAdvancedTestOvn(
         #       when a VM is created, it is a known limitation.
         #       Therefore VM's dns-name/hostname is checked to be as VM's name.
 
-        # TODO(mblue): implement alternative for next gen
         # ensures overcloud nodes are up for next tests
-        self.skipException("Powering nodes not supported yet (TODO)")
+        self.addCleanup(self.ensure_overcloud_nodes_active)
-        # self.addCleanup(self.ensure_overcloud_nodes_active)
         # create port with dns-name (as VM name)
         vm_name = self._rand_name('vm')
         dns_port = self.create_port(self.network,
@@ -347,19 +353,11 @@ class InternalDNSInterruptionsAdvancedTestOvn(
         vm_1['ssh_client'] = self._create_ssh_client(
             vm_1['fip']['floating_ip_address'])
         self._get_router_and_nodes_info()
-        # TODO(mblue): implement alternative for next gen
-        # node_id = self.hosts_info.get_overcloud_node_id(
-        #     self.router_gateway_chassis)
         # soft shutdown master networker node
-        # TODO(mblue): implement alternative for next gen
+        self.power_off_host(self.router_gateway_chassis)
-        # node_power.power_off(node_id)
         # validate hostname (dns-name) using API, guest VM,
         # and OVN NBDB when networker node is off and on
         self._dns_all_validations(vm_name, dns_port, vm_1['ssh_client'])
         # turn on networker node, wait until it is up and working
-        # TODO(mblue): implement alternative for next gen
+        self.power_on_host(self.router_gateway_chassis)
-        # node_power.power_on(
-        #     node_id,
-        #     services_clients=[self.os_admin.network_client],
-        #     agents_clients=[self.os_admin.compute.ServicesClient()])
         self._dns_all_validations(vm_name, dns_port, vm_1['ssh_client'])