add tripleo containers health checks

and use it in compute reboot test
done refactoring, unit tests are pasing , system tests are passing

related bz:
https://bugzilla.redhat.com/show_bug.cgi?id=1797892

Change-Id: I2816d03c26e8eeca92c99116e28be52f98aa5141

tobiko/docker/_client.py

@@ -30,7 +30,7 @@ def get_docker_client(base_urls=None, ssh_client=None):
 
 def list_docker_containers(client=None, **kwargs):
     try:
-        containers = docker_client(client).containers.list(**kwargs)
+        containers = docker_client(client).containers.list(all=True, **kwargs)
     except _exception.DockerUrlNotFoundError:
         return tobiko.Selection()
     else:

tobiko/openstack/topology/_topology.py

@@ -26,6 +26,7 @@ from six.moves.urllib import parse
 
 import tobiko
 from tobiko import docker
+from tobiko import podman
 from tobiko.shell import ip
 from tobiko.shell import ping
 from tobiko.shell import sh
@@ -97,6 +98,7 @@ def set_default_openstack_topology_class(topology_class):
 class OpenStackTopologyNode(object):
 
     _docker_client = None
+    _podman_client = None
 
     def __init__(self, topology, name, public_ip, ssh_client):
         self._topology = weakref.ref(topology)
@@ -124,6 +126,14 @@ class OpenStackTopologyNode(object):
                 ssh_client=self.ssh_client)
         return docker_client
 
+    @property
+    def podman_client(self):
+        podman_client = self._podman_client
+        if not podman_client:
+            self._podman_client = podman_client = podman.get_podman_client(
+                ssh_client=self.ssh_client)
+        return podman_client
+
     def __repr__(self):
         return "{cls!s}<name={name!r}>".format(cls=type(self).__name__,
                                                name=self.name)

tobiko/podman/_client.py

@@ -19,6 +19,7 @@ import six
 
 import podman
 
+
 import tobiko
 from tobiko.podman import _exception
 from tobiko.podman import _shell
@@ -76,28 +77,31 @@ class PodmanClientFixture(tobiko.SharedFixture):
 
     def setup_client(self):
         # setup podman access via varlink
-        podman_client_setup_cmds = [
-            "sudo groupadd -f podman",
-            "sudo usermod -a -G podman heat-admin",
-            "sudo chmod o+w /etc/tmpfiles.d",
-            "sudo echo 'd /run/podman 0750 root heat-admin' > "
-            "/etc/tmpfiles.d/podman.conf",
-            "sudo cp /lib/systemd/system/io.podman.socket /etc/systemd/system/"
-            "io.podman.socket",
-            "sudo crudini --set /etc/systemd/system/io.podman.socket Socket "
-            "SocketMode 0660",
-            "sudo crudini --set /etc/systemd/system/io.podman.socket Socket"
-            " SocketGroup podman",
-            "sudo systemctl daemon-reload",
-            "sudo systemd-tmpfiles --create",
-            "sudo systemctl enable --now io.podman.socket",
-            "sudo chown -R root: /run/podman",
-            "sudo chmod g+rw /run/podman/io.podman",
-            "sudo systemctl start io.podman.socket"
-        ]
+        podman_client_setup_cmds = \
+            "sudo test -f /var/varlink_client_access_setup ||  \
+            (sudo groupadd -f podman &&  \
+            sudo usermod -a -G podman heat-admin && \
+            sudo chmod -R o=wxr /etc/tmpfiles.d && \
+            sudo echo 'd /run/podman 0770 root heat-admin' >  \
+            /etc/tmpfiles.d/podman.conf && \
+            sudo cp /lib/systemd/system/io.podman.socket \
+            /etc/systemd/system/io.podman.socket && \
+            sudo crudini --set /etc/systemd/system/io.podman.socket Socket  \
+            SocketMode 0660 && \
+            sudo crudini --set /etc/systemd/system/io.podman.socket Socket  \
+            SocketGroup podman && \
+            sudo systemctl daemon-reload && \
+            sudo systemd-tmpfiles --create && \
+            sudo systemctl enable --now io.podman.socket && \
+            sudo chmod 777 /run/podman && \
+            sudo chown -R root: /run/podman && \
+            sudo chmod g+rw /run/podman/io.podman && \
+            sudo chmod 777 /run/podman/io.podman && \
+            sudo setenforce 0 && \
+            sudo systemctl start io.podman.socket && \
+            sudo touch /var/varlink_client_access_setup)"
 
-        for cmd in podman_client_setup_cmds:
-            sh.execute(cmd, ssh_client=self.ssh_client)
+        sh.execute(podman_client_setup_cmds, ssh_client=self.ssh_client)
 
         client = self.client
         if client is None:
@@ -105,19 +109,25 @@ class PodmanClientFixture(tobiko.SharedFixture):
         return client
 
     def create_client(self):
-        podman_remote_socket = self.discover_podman_socket()
-        podman_remote_socket_uri = 'unix:/tmp/podman.sock'
+        for _ in range(360):
 
-        remote_uri = 'ssh://{username}@{host}{socket}'.format(
-            username=self.ssh_client.connect_parameters['username'],
-            host=self.ssh_client.connect_parameters["hostname"],
-            socket=podman_remote_socket)
+            try:
+                podman_remote_socket = self.discover_podman_socket()
+                podman_remote_socket_uri = 'unix:/tmp/podman.sock'
 
-        client = podman.Client(uri=podman_remote_socket_uri,
-                               remote_uri=remote_uri,
-                               identity_file='~/.ssh/id_rsa')
-        client.system.ping()
-        return client
+                remote_uri = 'ssh://{username}@{host}{socket}'.format(
+                    username=self.ssh_client.connect_parameters['username'],
+                    host=self.ssh_client.connect_parameters["hostname"],
+                    socket=podman_remote_socket)
+
+                client = podman.Client(uri=podman_remote_socket_uri,
+                                       remote_uri=remote_uri,
+                                       identity_file='~/.ssh/id_rsa')
+                client.system.ping()
+                return client
+            except (ConnectionRefusedError, ConnectionResetError):
+                # retry
+                self.create_client()
 
     def connect(self):
         return tobiko.setup_fixture(self).client

tobiko/tests/faults/ha/test_cloud_recovery.py

@@ -6,6 +6,7 @@ from tobiko.shell import sh
 from tobiko.tests.faults.ha import cloud_disruptions
 from tobiko.tripleo import pacemaker
 from tobiko.tripleo import processes
+from tobiko.tripleo import containers
 from tobiko.openstack import stacks
 import tobiko
 
@@ -17,13 +18,12 @@ def nodes_health_check():
 
     # TODO:
     # Test existing created servers
-    # ServerStackResourcesTest().test_server_create()
 
 
 # check vm create with ssh and ping checks
 def check_vm_create(stack_name):
-    '''stack_name: unique stack name ,
-    so that each time a new vm is created'''
+    """stack_name: unique stack name ,
+    so that each time a new vm is created"""
     # create a vm
     stack = stacks.CirrosServerStackFixture(
         stack_name=stack_name)
@@ -61,8 +61,14 @@ class RebootNodesTest(testtools.TestCase):
 
     def test_reboot_computes_recovery(self):
         nodes_health_check()
+        computes_containers_dict_before = \
+            containers.list_containers(group='compute')
         cloud_disruptions.reset_all_compute_nodes(hard_reset=True)
         nodes_health_check()
+        computes_containers_dict_after = \
+            containers.list_containers(group='compute')
+        containers.assert_equal_containers_state(
+            computes_containers_dict_before, computes_containers_dict_after)
         check_vm_create(stack_name=self.id())
 
 # [..]

tobiko/tripleo/containers.py

@@ -0,0 +1,136 @@
+from __future__ import absolute_import
+
+from oslo_log import log
+import pandas
+
+import tobiko
+from tobiko import podman
+from tobiko import docker
+from tobiko.openstack import topology
+
+
+LOG = log.getLogger(__name__)
+
+
+def container_runtime():
+    """check what container runtime is running
+    and return a handle to it"""
+
+    ssh_client = topology.list_openstack_nodes(group='controller')[
+        0].ssh_client
+    if docker.is_docker_running(ssh_client=ssh_client):
+        return docker
+    else:
+        return podman
+
+
+container_runtime_type = container_runtime()
+
+
+def list_node_containers(client=None):
+    """returns a list of containers and their run state"""
+
+    if container_runtime_type == podman:
+        return container_runtime_type.list_podman_containers(client=client)
+
+    elif container_runtime_type == docker:
+        return container_runtime_type.list_docker_containers(client=client)
+
+
+def get_container_client(ssh_client=None):
+    """returns a list of containers and their run state"""
+
+    if container_runtime_type == podman:
+        return container_runtime_type.get_podman_client(
+            ssh_client=ssh_client).connect()
+
+    elif container_runtime_type == docker:
+        return container_runtime_type.get_docker_client(
+            ssh_client=ssh_client).connect()
+
+
+def list_containers(group=None):
+    """get list of containers in running state
+    from specified node group
+    returns : a list of overcloud_node's running containers"""
+
+    # moved here from topology
+    # reason : Workaround for :
+    # AttributeError: module 'tobiko.openstack.topology' has no
+    # attribute 'container_runtime'
+
+    containers_list = tobiko.Selection()
+    openstack_nodes = topology.list_openstack_nodes(group=group)
+
+    for node in openstack_nodes:
+        ssh_client = node.ssh_client
+        container_client = get_container_client(ssh_client)
+        node_containers_list = list_node_containers(client=container_client)
+        containers_list.extend(node_containers_list)
+    return containers_list
+
+
+def comparable_container_keys(container):
+    """returns the tuple : 'container_host','container_name',
+    'container_state'
+     """
+    if container_runtime_type == podman:
+        return (container._client._context.hostname,  # pylint: disable=W0212
+                container.data['names'], container.data['status'])
+
+    elif container_runtime_type == docker:
+        return (container.attrs['Config']['Hostname'],
+                container.attrs['State']['Status'], container.attrs['Name'])
+
+
+def get_container_states_list(containers_list):
+    container_states_list = tobiko.Selection()
+    container_states_list.extend([comparable_container_keys(container) for
+                                  container in containers_list])
+    return container_states_list
+
+
+def dataframe_difference(df1, df2, which=None):
+    """Find rows which are different between two DataFrames."""
+    comparison_df = df1.merge(df2,
+                              indicator='same_state',
+                              how='outer')
+    if which is None:
+        diff_df = comparison_df[comparison_df['same_state'] != 'both']
+    else:
+        diff_df = comparison_df[comparison_df['same_state'] == which]
+    return diff_df
+
+
+def assert_equal_containers_state(expected_containers_list,
+                                  actual_containers_list):
+
+    """compare container with states from two lists"""
+
+    failures = []
+    expected_containers_list_df = pandas.DataFrame(
+        get_container_states_list(expected_containers_list),
+        columns=['container_host', 'container_name', 'container_state'])
+    actual_containers_list_df = pandas.DataFrame(
+        get_container_states_list(actual_containers_list),
+        columns=['container_host', 'container_name', 'container_state'])
+
+    LOG.info('expected_containers_list_df: {} '.format(
+        expected_containers_list_df.to_string(index=False)))
+    LOG.info('actual_containers_list_df: {} '.format(
+        actual_containers_list_df.to_string(index=False)))
+
+    # execute a dataframe diff between the excpected and actual containers
+    expected_containers_state_changed = \
+        dataframe_difference(expected_containers_list_df,
+                             actual_containers_list_df)
+    # check for changed state containers
+    if not expected_containers_state_changed.empty:
+        failures.append('expected containers changed state ! : \n\n{}'.format(
+            expected_containers_state_changed.to_string(index=False)))
+
+    if failures:
+        tobiko.fail('container states mismatched:\n{!s}', '\n'.join(failures))
+    else:
+        LOG.info("assert_equal_containers_state :"
+                 " OK, all containers are on the same state")