[Podified] Make Topology class to decide about container runtime

Previously it was only used in the Tripleo Topology class and containers
module decided about runtime (docker or podman) which should be used.
Now we have also Podified deployments and we can't rely on checks like
"has_overcloud()" to decide about it.

This patch moves some containers related things to the rhosp and/or
podified modules and let topology class to decide what containers
runtime should be used.

Related-Jira: #OSP-22166
Change-Id: I7187de8884e44eaefcdda6fae160e89242761f39

tobiko/openstack/topology/__init__.py

@@ -63,3 +63,5 @@ set_default_openstack_topology_class = (
     _topology.set_default_openstack_topology_class)
 verify_osp_version = _topology.verify_osp_version
 get_config_setting = _topology.get_config_setting
+node_name_from_hostname = _topology.node_name_from_hostname
+remove_duplications = _topology.remove_duplications

tobiko/openstack/topology/_topology.py

@@ -311,6 +311,8 @@ class OpenStackTopology(tobiko.SharedFixture):
 
     has_containers = False
 
+    container_runtime_cmd = 'docker'
+
     config_file_mappings = {
         'ml2_conf.ini': '/etc/neutron/plugins/ml2/ml2_conf.ini',
         'bgp-agent.conf': '/etc/ovn-bgp-agent/bgp-agent.conf'
@@ -396,6 +398,15 @@ class OpenStackTopology(tobiko.SharedFixture):
                             ssh_client=node.ssh_client)
         return digger
 
+    def assert_containers_running(self, expected_containers,
+                                  group=None,
+                                  full_name=True, bool_check=False,
+                                  nodenames=None):
+        pass
+
+    def list_containers_df(self, group=None):
+        pass
+
     def discover_nodes(self):
         self.discover_ssh_proxy_jump_node()
         self.discover_configured_nodes()

tobiko/podified/_topology.py

@@ -22,6 +22,7 @@ from tobiko.openstack import neutron
 from tobiko.openstack import topology
 from tobiko.podified import _edpm
 from tobiko.podified import _openshift
+from tobiko.podified import containers
 from tobiko import rhosp
 from tobiko.shell import ssh
 
@@ -67,9 +68,29 @@ class PodifiedTopology(rhosp.RhospTopology):
         neutron.FRR: 'frr'
     }
 
-    def __init__(self):
+    sidecar_container_list = [
-        super(PodifiedTopology, self).__init__()
+        'neutron-haproxy-ovnmeta',
-        self.ocp_workers = {}
+        'neutron-dnsmasq-qdhcp'
+    ]
+
+    @property
+    def ignore_containers_list(self):
+        return self.sidecar_container_list
+
+    def assert_containers_running(self, expected_containers,
+                                  group=None,
+                                  full_name=True, bool_check=False,
+                                  nodenames=None):
+        group = group or ALL_COMPUTES_GROUP_NAME
+        return containers.assert_containers_running(
+            group=group,
+            expected_containers=expected_containers,
+            full_name=full_name,
+            bool_check=bool_check,
+            nodenames=nodenames)
+
+    def list_containers_df(self, group=None):
+        return containers.list_containers_df(group)
 
     def add_node(self,
                  hostname: typing.Optional[str] = None,

tobiko/podified/containers.py

@@ -0,0 +1,351 @@
+from __future__ import absolute_import
+
+import functools
+import os
+import time
+import typing
+
+from oslo_log import log
+import pandas
+
+import tobiko
+from tobiko import config
+from tobiko.openstack import neutron
+from tobiko.openstack import topology
+from tobiko import rhosp as rhosp_topology
+from tobiko.rhosp import containers as rhosp_containers
+
+
+CONF = config.CONF
+LOG = log.getLogger(__name__)
+
+
+class ContainerRuntimeFixture(tobiko.SharedFixture):
+
+    runtime: typing.Optional[rhosp_containers.ContainerRuntime] = None
+
+    def setup_fixture(self):
+        self.runtime = self.get_runtime()
+
+    def cleanup_fixture(self):
+        self.runtime = None
+
+    @staticmethod
+    def get_runtime() -> typing.Optional[rhosp_containers.ContainerRuntime]:
+        """return handle to the container runtime"""
+        return rhosp_containers.PODMAN_RUNTIME
+
+
+def get_container_runtime() -> rhosp_containers.ContainerRuntime:
+    runtime = tobiko.setup_fixture(ContainerRuntimeFixture).runtime
+    return runtime
+
+
+def get_container_runtime_name() -> str:
+    return get_container_runtime().runtime_name
+
+
+def is_docker() -> bool:
+    return False
+
+
+def is_podman() -> bool:
+    return True
+
+
+def has_container_runtime() -> bool:
+    return True
+
+
+@functools.lru_cache()
+def list_node_containers(ssh_client):
+    """returns a list of containers and their run state"""
+    return get_container_runtime().list_containers(ssh_client=ssh_client)
+
+
+def get_container_client(ssh_client=None):
+    """returns a list of containers and their run state"""
+    return get_container_runtime().get_client(ssh_client=ssh_client)
+
+
+def list_containers_df(group=None):
+    actual_containers_list = list_containers(group)
+    return pandas.DataFrame(
+        get_container_states_list(actual_containers_list),
+        columns=['container_host', 'container_name', 'container_state'])
+
+
+def list_containers(group=None):
+    """get list of containers in running state
+    from specified node group
+    returns : a list of overcloud_node's running containers"""
+
+    # moved here from topology
+    # reason : Workaround for :
+    # AttributeError: module 'tobiko.openstack.topology' has no
+    # attribute 'container_runtime'
+
+    if group is None:
+        group = 'compute'
+    containers_list = tobiko.Selection()
+    openstack_nodes = topology.list_openstack_nodes(group=group)
+
+    for node in openstack_nodes:
+        LOG.debug(f"List containers for node {node.name}")
+        node_containers_list = list_node_containers(ssh_client=node.ssh_client)
+        containers_list.extend(node_containers_list)
+    return containers_list
+
+
+expected_containers_file = os.path.expanduser(
+    '~/expected_containers_list_df.csv')
+
+
+def save_containers_state_to_file(expected_containers_list,):
+    expected_containers_list_df = pandas.DataFrame(
+        get_container_states_list(expected_containers_list),
+        columns=['container_host', 'container_name', 'container_state'])
+    expected_containers_list_df.to_csv(
+        expected_containers_file)
+    return expected_containers_file
+
+
+def assert_containers_running(group, expected_containers, full_name=True,
+                              bool_check=False, nodenames=None):
+
+    """assert that all containers specified in the list are running
+    on the specified openstack group(controller or compute etc..)
+    if bool_check is True then return only True or false without failing"""
+
+    if is_docker():
+        LOG.info('not checking common containers since we are on docker')
+        return
+
+    failures = []
+
+    openstack_nodes = topology.list_openstack_nodes(group=group,
+                                                    hostnames=nodenames)
+    for node in openstack_nodes:
+        node_containers = list_node_containers(ssh_client=node.ssh_client)
+        containers_list_df = pandas.DataFrame(
+            get_container_states_list(node_containers),
+            columns=['container_host', 'container_name', 'container_state'])
+        # check that the containers are present
+        LOG.info('node: {} containers list : {}'.format(
+            node.name, containers_list_df.to_string(index=False)))
+        for container in expected_containers:
+            # get container attrs dataframe
+            if full_name:
+                container_attrs = containers_list_df.query(
+                    'container_name == "{}"'.format(container))
+            else:
+                container_attrs = containers_list_df[
+                    containers_list_df['container_name'].
+                    str.contains(container)]
+            # check if the container exists
+            LOG.info('checking container: {}'.format(container))
+            if container_attrs.empty:
+                failures.append(
+                    'expected container {} not found on node {} ! : \n\n'.
+                    format(container, node.name))
+            # if container exists, check it is running
+            else:
+                # only one running container is expected
+                container_running_attrs = container_attrs.query(
+                    'container_state=="running"')
+                if container_running_attrs.empty:
+                    failures.append(
+                        'expected container {} is not running on node {} , '
+                        'its state is {}! : \n\n'.format(
+                            container, node.name,
+                            container_attrs.container_state.values.item()))
+                elif len(container_running_attrs) > 1:
+                    failures.append(
+                        'only one running container {} was expected on '
+                        'node {}, but got {}! : \n\n'.format(
+                            container, node.name,
+                            len(container_running_attrs)))
+
+    if not bool_check and failures:
+        tobiko.fail(
+            'container states mismatched:\n{}'.format('\n'.join(failures)),
+            rhosp_containers.ContainerMismatchException)
+
+    elif bool_check and failures:
+        return False
+
+    else:
+        LOG.info('All specified containers are in running state! ')
+        return True
+
+
+def assert_ovn_containers_running():
+    if not neutron.has_ovn():
+        LOG.info("Networking OVN not configured")
+        return
+    ovn_containers = ['ovn_metadata_agent',
+                      'ovn_controller']
+    groups = ['edpm-compute', 'edpm-networker']
+    for group in groups:
+        assert_containers_running(group, ovn_containers, full_name=False)
+    LOG.info("Networking OVN containers verified in running state")
+
+
+def get_container_states_list(containers_list,
+                              include_container_objects=False):
+    container_states_list = tobiko.Selection()
+    container_states_list.extend([comparable_container_keys(
+        container, include_container_objects=include_container_objects) for
+                                  container in containers_list])
+    return container_states_list
+
+
+def comparable_container_keys(container, include_container_objects=False):
+    """returns the tuple : 'container_host','container_name',
+    'container_state, container object if specified'
+     """
+    # Differenciate between podman_ver3 with podman-py from earlier api
+    if include_container_objects:
+        return (rhosp_topology.ip_to_hostname(
+                    container.client.base_url.netloc.rsplit('_')[1]),
+                container.attrs['Names'][0], container.attrs['State'],
+                container)
+    else:
+        return (rhosp_topology.ip_to_hostname(
+                    container.client.base_url.netloc.rsplit('_')[1]),
+                container.attrs['Names'][0],
+                container.attrs['State'])
+
+
+@functools.lru_cache()
+def list_containers_objects_df():
+    containers_list = list_containers()
+    containers_objects_list_df = pandas.DataFrame(
+        get_container_states_list(
+            containers_list, include_container_objects=True),
+        columns=['container_host', 'container_name',
+                 'container_state', 'container_object'])
+    return containers_objects_list_df
+
+
+def get_edpm_container(container_name=None, container_host=None,
+                       partial_container_name=None):
+    """gets an container object by name on specified host
+    container"""
+    con_obj_df = list_containers_objects_df()
+    if partial_container_name and container_host:
+        con_obj_df = con_obj_df[con_obj_df['container_name'].str.contains(
+            partial_container_name)]
+        contaniner_obj = con_obj_df.query(
+            'container_host == "{container_host}"'.format(
+                container_host=container_host))['container_object']
+    elif container_host:
+        contaniner_obj = con_obj_df.query(
+            'container_name == "{container_name}"'
+            ' and container_host == "{container_host}"'.
+            format(container_host=container_host,
+                   container_name=container_name)).container_object
+    else:
+        contaniner_obj = con_obj_df.query(
+            'container_name == "{container_name}"'.
+            format(container_name=container_name)).container_object
+    if not contaniner_obj.empty:
+        return contaniner_obj.values[0]
+    else:
+        tobiko.fail('container {} not found!'.format(container_name))
+
+
+def action_on_container(action: str,
+                        container_name=None,
+                        container_host=None,
+                        partial_container_name=None):
+    """take a container and perform an action on it
+    actions are as defined in : podman/libs/containers.py:14/164"""
+
+    LOG.debug(f"Executing '{action}' action on container "
+              f"'{container_name}@{container_host}'...")
+    container = get_edpm_container(
+        container_name=container_name,
+        container_host=container_host,
+        partial_container_name=partial_container_name)
+
+    container_class: typing.Type = type(container)
+    # we get the specified action as function from podman lib
+    action_method: typing.Optional[typing.Callable] = getattr(
+        container_class, action, None)
+    if action_method is None:
+        raise TypeError(f"Unsupported container action for class :"
+                        f" {container_class}")
+    if not callable(action_method):
+        raise TypeError(
+            f"Attribute '{container_class.__qualname__}.{action}' value "
+            f" is not a method: {action_method!r}")
+    LOG.debug(f"Calling '{action_method}' action on container "
+              f"'{container}'")
+    return action_method(container)
+
+
+def assert_equal_containers_state(expected_containers_list=None,
+                                  timeout=120, interval=2,
+                                  recreate_expected=False):
+
+    """compare all edpm container states with using two lists:
+    one is current , the other some past list
+    first time this method runs it creates a file holding overcloud
+    containers' states: ~/expected_containers_list_df.csv'
+    second time it creates a current containers states list and
+    compares them, they must be identical"""
+
+    # if we have a file or an explicit variable use that , otherwise  create
+    # and return
+    if recreate_expected or (not expected_containers_list and
+                             not os.path.exists(expected_containers_file)):
+        save_containers_state_to_file(list_containers())
+        return
+
+    elif expected_containers_list:
+        expected_containers_list_df = pandas.DataFrame(
+            get_container_states_list(expected_containers_list),
+            columns=['container_host', 'container_name', 'container_state'])
+
+    elif os.path.exists(expected_containers_file):
+        expected_containers_list_df = pandas.read_csv(
+            expected_containers_file)
+
+    failures = []
+    start = time.time()
+    error_info = 'Output explanation: left_only is the original state, ' \
+                 'right_only is the new state'
+
+    while time.time() - start < timeout:
+
+        failures = []
+        actual_containers_list_df = list_containers_df()
+
+        LOG.info('expected_containers_list_df: {} '.format(
+            expected_containers_list_df.to_string(index=False)))
+        LOG.info('actual_containers_list_df: {} '.format(
+            actual_containers_list_df.to_string(index=False)))
+
+        # execute a `dataframe` diff between the expected and actual containers
+        expected_containers_state_changed = \
+            rhosp_containers.dataframe_difference(
+                    expected_containers_list_df,
+                    actual_containers_list_df)
+        # check for changed state containerstopology
+        if not expected_containers_state_changed.empty:
+            failures.append('expected containers changed state ! : '
+                            '\n\n{}\n{}'.format(
+                             expected_containers_state_changed.
+                             to_string(index=False), error_info))
+            LOG.info('container states mismatched:\n{}\n'.format(failures))
+            time.sleep(interval)
+            # clear cache to obtain new data
+            list_node_containers.cache_clear()
+        else:
+            LOG.info("assert_equal_containers_state :"
+                     " OK, all containers are on the same state")
+            return
+    if failures:
+        tobiko.fail('container states mismatched:\n{!s}', '\n'.join(
+            failures))

tobiko/rhosp/__init__.py

@@ -21,3 +21,5 @@ RhospNode = _topology.RhospNode
 
 get_rhosp_release = _version_utils.get_rhosp_release
 get_rhosp_version = _version_utils.get_rhosp_version
+
+ip_to_hostname = _topology.ip_to_hostname

tobiko/rhosp/_topology.py

@@ -27,6 +27,27 @@ from tobiko.shell import ssh
 LOG = log.getLogger(__name__)
 
 
+def get_ip_to_nodes_dict(group, openstack_nodes=None):
+    if not openstack_nodes:
+        openstack_nodes = topology.list_openstack_nodes(group=group)
+    ip_to_nodes_dict = {str(node.public_ip): node.name for node in
+                        openstack_nodes}
+    return ip_to_nodes_dict
+
+
+def ip_to_hostname(oc_ip, group=None):
+    ip_to_nodes_dict = get_ip_to_nodes_dict(group)
+    oc_ipv6 = oc_ip.replace(".", ":")
+    if netaddr.valid_ipv4(oc_ip) or netaddr.valid_ipv6(oc_ip):
+        return ip_to_nodes_dict[oc_ip]
+    elif netaddr.valid_ipv6(oc_ipv6):
+        LOG.debug("The provided string was a modified IPv6 address: %s",
+                  oc_ip)
+        return ip_to_nodes_dict[oc_ipv6]
+    else:
+        tobiko.fail("wrong IP value provided %s" % oc_ip)
+
+
 class RhospTopology(topology.OpenStackTopology):
 
     """Base topology for Red Hat OpenStack deployments.
@@ -36,6 +57,11 @@ class RhospTopology(topology.OpenStackTopology):
     """
 
     has_containers = True
+    container_runtime_cmd = 'podman'
+
+    @property
+    def ignore_containers_list(self):
+        return None
 
 
 class RhospNode(topology.OpenStackTopologyNode):

tobiko/rhosp/containers.py

@@ -0,0 +1,123 @@
+from __future__ import absolute_import
+
+import abc
+import re
+import typing
+
+from oslo_log import log
+
+import tobiko
+from tobiko.openstack import topology
+from tobiko import podman
+from tobiko import docker
+from tobiko.shell import ssh
+
+
+LOG = log.getLogger(__name__)
+
+
+class ContainerRuntime(abc.ABC):
+    runtime_name: str
+    version_pattern: typing.Pattern
+
+    def match_version(self, version: str) -> bool:
+        for version_line in version.splitlines():
+            if self.version_pattern.match(version_line) is not None:
+                return True
+        return False
+
+    def get_client(self, ssh_client):
+        for attempt in tobiko.retry(timeout=60.0,
+                                    interval=5.0):
+            try:
+                client = self._get_client(ssh_client=ssh_client)
+                break
+            # TODO chose a better exception type
+            except Exception:
+                if attempt.is_last:
+                    raise
+                LOG.debug('Unable to connect to docker server',
+                          exc_info=1)
+                ssh.reset_default_ssh_port_forward_manager()
+        else:
+            raise RuntimeError("Broken retry loop")
+        return client
+
+    def _get_client(self, ssh_client):
+        raise NotImplementedError
+
+    def list_containers(self, ssh_client):
+        raise NotImplementedError
+
+
+class DockerContainerRuntime(ContainerRuntime):
+    runtime_name = 'docker'
+    version_pattern = re.compile('Docker version .*', re.IGNORECASE)
+
+    def _get_client(self, ssh_client):
+        return docker.get_docker_client(ssh_client=ssh_client,
+                                        sudo=True).connect()
+
+    def list_containers(self, ssh_client):
+        client = self.get_client(ssh_client=ssh_client)
+        return docker.list_docker_containers(client=client)
+
+
+class PodmanContainerRuntime(ContainerRuntime):
+    runtime_name = 'podman'
+    version_pattern = re.compile('Podman version .*', re.IGNORECASE)
+
+    def _get_client(self, ssh_client):
+        return podman.get_podman_client(ssh_client=ssh_client).connect()
+
+    def list_containers(self, ssh_client):
+        client = self.get_client(ssh_client=ssh_client)
+        return podman.list_podman_containers(client=client)
+
+
+DOCKER_RUNTIME = DockerContainerRuntime()
+PODMAN_RUNTIME = PodmanContainerRuntime()
+CONTAINER_RUNTIMES = [PODMAN_RUNTIME, DOCKER_RUNTIME]
+
+
+class ContainerMismatchException(tobiko.TobikoException):
+    pass
+
+
+def remove_containers_from_comparison(comparable_containers_df):
+    """remove any containers if comparing them with previous status is not
+    necessary or makes no sense
+    """
+    os_topology = topology.get_openstack_topology()
+    for row in comparable_containers_df.iterrows():
+        for ignore_container in os_topology.ignore_containers_list:
+            if ignore_container in str(row):
+                LOG.info(f'container {ignore_container} has changed state, '
+                         'but that\'s ok - it will be ignored and the test '
+                         f'will not fail due to this: {str(row)}')
+                # if a pcs resource is found , we drop that row
+                comparable_containers_df.drop(row[0], inplace=True)
+                # this row was already dropped, go to next row
+                break
+
+
+def dataframe_difference(df1, df2, which=None):
+    """Find rows which are different between two DataFrames."""
+    comparison_df = df1.merge(df2,
+                              indicator='same_state',
+                              how='outer')
+    # return only non identical rows
+    if which is None:
+        diff_df = comparison_df[comparison_df['same_state'] != 'both']
+
+    else:
+        diff_df = comparison_df[comparison_df['same_state'] == which]
+
+    # if the list of different state containers includes sidecar containers,
+    # ignore them because the existence of these containers depends on the
+    # created resources
+    # if the list of different state containers includes pacemaker resources,
+    # ignore them since the sanity and fault tests check pacemaker status too
+    remove_containers_from_comparison(diff_df)
+
+    return diff_df

tobiko/tests/faults/neutron/test_agents.py

@@ -31,8 +31,6 @@ from tobiko.openstack import tests
 from tobiko.openstack import topology
 from tobiko.shell import ping
 from tobiko.shell import sh
-from tobiko.tripleo import containers
-from tobiko.tripleo import overcloud
 
 
 LOG = log.getLogger(__name__)
@@ -69,10 +67,7 @@ class BaseAgentTest(testtools.TestCase):
 
     @property
     def container_runtime_name(self):
-        if overcloud.has_overcloud():
+        return topology.get_openstack_topology().container_runtime_cmd
-            return containers.get_container_runtime_name()
-        else:
-            return 'docker'
 
     def get_agent_container_name(self, agent_name):
         try:
@@ -88,7 +83,8 @@ class BaseAgentTest(testtools.TestCase):
                 self.agent_name)
             if not self.container_name:
                 return
-            oc_containers_df = containers.list_containers_df().query(
+            oc_containers_df = \
+                topology.get_openstack_topology().list_containers_df().query(
                     f'container_name == "{self.container_name}"')
             LOG.debug(
                 f"{self.container_name} container found:\n{oc_containers_df}")
@@ -166,8 +162,7 @@ class BaseAgentTest(testtools.TestCase):
         '''
         self._do_agent_action(START, hosts)
         if self.container_name:
-            containers.assert_containers_running(
+            topology.get_openstack_topology().assert_containers_running(
-                group='overcloud',
                 expected_containers=[self.container_name],
                 nodenames=hosts or self.hosts)
 
@@ -181,8 +176,7 @@ class BaseAgentTest(testtools.TestCase):
         '''
         self._do_agent_action(RESTART, hosts)
         if self.container_name:
-            containers.assert_containers_running(
+            topology.get_openstack_topology().assert_containers_running(
-                group='overcloud',
                 expected_containers=[self.container_name],
                 nodenames=hosts or self.hosts)
 
@@ -213,8 +207,7 @@ class BaseAgentTest(testtools.TestCase):
                        sudo=True)
 
         if self.container_name:
-            containers.assert_containers_running(
+            topology.get_openstack_topology().assert_containers_running(
-                group='overcloud',
                 expected_containers=[self.container_name],
                 nodenames=hosts or self.hosts)
 

tobiko/tests/functional/tripleo/test_containers.py

@@ -15,6 +15,7 @@ from __future__ import absolute_import
 
 import testtools
 
+from tobiko.rhosp import containers as rhosp_containers
 from tobiko.tripleo import containers
 
 
@@ -23,7 +24,7 @@ class RuntimeRuntimeTest(testtools.TestCase):
 
     def test_get_container_runtime(self):
         runtime = containers.get_container_runtime()
-        self.assertIsInstance(runtime, containers.ContainerRuntime)
+        self.assertIsInstance(runtime, rhosp_containers.ContainerRuntime)
 
     def test_list_containers(self):
         containers_list = containers.list_containers()

tobiko/tests/sanity/containers/test_containers.py

@@ -26,6 +26,7 @@ import pandas
 import tobiko
 from tobiko.openstack import neutron
 from tobiko.openstack import topology
+from tobiko.rhosp import containers as rhosp_containers
 from tobiko.shell import sh
 from tobiko import tripleo
 from tobiko.tripleo import containers
@@ -34,8 +35,15 @@ from tobiko.tripleo import containers
 LOG = log.getLogger(__name__)
 
 
+class BaseContainersHealtTest(testtools.TestCase):
+
+    def _assert_containers_running(self, group, expected_containers):
+        topology.get_openstack_topology().assert_containers_running(
+            group=group, expected_containers=expected_containers)
+
+
 @tripleo.skip_if_missing_overcloud
-class ContainersHealthTest(testtools.TestCase):
+class ContainersHealthTest(BaseContainersHealtTest):
     # TODO(eolivare): refactor this class, because it replicates some code from
     # tobiko/tripleo/containers.py and its tests may be duplicating what
     # test_0vercloud_health_check already covers when it calls
@@ -49,158 +57,150 @@ class ContainersHealthTest(testtools.TestCase):
 
     def test_cinder_api(self):
         """check that all common tripleo containers are running"""
-        containers.assert_containers_running('controller', ['cinder_api'])
+        self._assert_containers_running('controller', ['cinder_api'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_rsync(self):
-        containers.assert_containers_running('controller', ['swift_rsync'])
+        self._assert_containers_running('controller', ['swift_rsync'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_proxy(self):
-        containers.assert_containers_running('controller', ['swift_proxy'])
+        self._assert_containers_running('controller', ['swift_proxy'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_object_updater(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller', ['swift_object_updater'])
-                                             ['swift_object_updater'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_object_server(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller', ['swift_object_server'])
-                                             ['swift_object_server'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_object_replicator(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller',
                                         ['swift_object_replicator'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_object_expirer(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller', ['swift_object_expirer'])
-                                             ['swift_object_expirer'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_object_auditor(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller', ['swift_object_auditor'])
-                                             ['swift_object_auditor'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_container_updater(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller',
                                         ['swift_container_updater'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_container_server(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller',
                                         ['swift_container_server'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_container_replicator(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller',
                                         ['swift_container_replicator'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_container_auditor(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller',
                                         ['swift_container_auditor'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_account_server(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller', ['swift_account_server'])
-                                             ['swift_account_server'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_account_replicator(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller',
                                         ['swift_account_replicator'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_account_reaper(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller', ['swift_account_reaper'])
-                                             ['swift_account_reaper'])
 
     @tripleo.skip_if_ceph_rgw()
     def test_swift_account_auditor(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller',
                                         ['swift_account_auditor'])
 
     def test_nova_vnc_proxy(self):
-        containers.assert_containers_running('controller', ['nova_vnc_proxy'])
+        self._assert_containers_running('controller', ['nova_vnc_proxy'])
 
     def test_nova_scheduler(self):
-        containers.assert_containers_running('controller', ['nova_scheduler'])
+        self._assert_containers_running('controller', ['nova_scheduler'])
 
     def test_nova_metadata(self):
-        containers.assert_containers_running('controller', ['nova_metadata'])
+        self._assert_containers_running('controller', ['nova_metadata'])
 
     def test_nova_conductor(self):
-        containers.assert_containers_running('controller', ['nova_conductor'])
+        self._assert_containers_running('controller', ['nova_conductor'])
 
     def test_nova_api_cron(self):
-        containers.assert_containers_running('controller', ['nova_api_cron'])
+        self._assert_containers_running('controller', ['nova_api_cron'])
 
     def test_nova_api(self):
-        containers.assert_containers_running('controller', ['nova_api'])
+        self._assert_containers_running('controller', ['nova_api'])
 
     def test_neutron_api(self):
-        containers.assert_containers_running('controller', ['neutron_api'])
+        self._assert_containers_running('controller', ['neutron_api'])
 
     def test_memcached(self):
-        containers.assert_containers_running('controller', ['memcached'])
+        self._assert_containers_running('controller', ['memcached'])
 
     def test_controller_logrotate_crond(self):
-        containers.assert_containers_running('controller', ['logrotate_crond'])
+        self._assert_containers_running('controller', ['logrotate_crond'])
 
     def test_keystone(self):
-        containers.assert_containers_running('controller', ['keystone'])
+        self._assert_containers_running('controller', ['keystone'])
 
     def test_controller_iscsid(self):
-        containers.assert_containers_running('controller', ['iscsid'])
+        self._assert_containers_running('controller', ['iscsid'])
 
     def test_horizon(self):
-        containers.assert_containers_running('controller', ['horizon'])
+        self._assert_containers_running('controller', ['horizon'])
 
     def test_heat_engine(self):
-        containers.assert_containers_running('controller', ['heat_engine'])
+        self._assert_containers_running('controller', ['heat_engine'])
 
     def test_heat_api_cron(self):
-        containers.assert_containers_running('controller', ['heat_api_cron'])
+        self._assert_containers_running('controller', ['heat_api_cron'])
 
     def test_heat_api_cfn(self):
-        containers.assert_containers_running('controller', ['heat_api_cfn'])
+        self._assert_containers_running('controller', ['heat_api_cfn'])
 
     def test_heat_api(self):
-        containers.assert_containers_running('controller', ['heat_api'])
+        self._assert_containers_running('controller', ['heat_api'])
 
     def test_glance_api(self):
-        containers.assert_containers_running('controller', ['glance_api'])
+        self._assert_containers_running('controller', ['glance_api'])
 
     def test_cinder_scheduler(self):
-        containers.assert_containers_running('controller',
+        self._assert_containers_running('controller', ['cinder_scheduler'])
-                                             ['cinder_scheduler'])
 
     def test_cinder_api_cron(self):
-        containers.assert_containers_running('controller', ['cinder_api_cron'])
+        self._assert_containers_running('controller', ['cinder_api_cron'])
 
     def test_compute_iscsid(self):
-        containers.assert_containers_running('compute', ['iscsid'])
+        self._assert_containers_running('compute', ['iscsid'])
 
     def test_compute_logrotate_crond(self):
-        containers.assert_containers_running('compute', ['logrotate_crond'])
+        self._assert_containers_running('compute', ['logrotate_crond'])
 
     def test_nova_compute(self):
-        containers.assert_containers_running('compute', ['nova_compute'])
+        self._assert_containers_running('compute', ['nova_compute'])
 
     def test_nova_libvirt(self):
         nova_libvirt = containers.get_libvirt_container_name()
-        containers.assert_containers_running('compute', [nova_libvirt])
+        self._assert_containers_running('compute', [nova_libvirt])
 
     def test_nova_migration_target(self):
-        containers.assert_containers_running('compute',
+        self._assert_containers_running('compute', ['nova_migration_target'])
-                                             ['nova_migration_target'])
 
     def test_nova_virtlogd(self):
-        containers.assert_containers_running('compute', ['nova_virtlogd'])
+        self._assert_containers_running('compute', ['nova_virtlogd'])
 
     def test_ovn_containers_running(self):
         containers.assert_ovn_containers_running()
@@ -244,8 +244,8 @@ class ContainersHealthTest(testtools.TestCase):
             # execute a `dataframe` diff between the expected
             # and actual containers
             expected_containers_state_changed = \
-                containers.dataframe_difference(expected_containers_list_df,
+                rhosp_containers.dataframe_difference(
-                                                actual_containers_list_df)
+                    expected_containers_list_df, actual_containers_list_df)
             # check for changed state containerstopology
             if not expected_containers_state_changed.empty:
                 failures.append('expected containers changed state ! : '

tobiko/tripleo/_topology.py

@@ -31,6 +31,7 @@ from tobiko.shell import sh
 from tobiko.shell import ssh
 from tobiko.tripleo import _overcloud
 from tobiko.tripleo import _undercloud
+from tobiko.tripleo import containers
 
 CONF = config.CONF
 LOG = log.getLogger(__name__)
@@ -84,12 +85,47 @@ class TripleoTopology(rhosp.RhospTopology):
         neutron.SERVER: '/var/log/containers/neutron/server.log*',
     }
 
+    _container_runtime_cmd = None
+
+    pcs_resource_list = ['haproxy', 'galera', 'redis', 'ovn-dbs', 'cinder',
+                         'rabbitmq', 'manila', 'ceph', 'pacemaker']
+
+    sidecar_container_list = [
+        'neutron-haproxy-ovnmeta', 'neutron-haproxy-qrouter',
+        'neutron-dnsmasq-qdhcp', 'neutron-keepalived-qrouter', 'neutron-radvd']
+
+    @property
+    def container_runtime_cmd(self):
+        if self._container_runtime_cmd is None:
+            self._container_runtime_cmd = \
+                    containers.get_container_runtime_name()
+        return self._container_runtime_cmd
+
+    @property
+    def ignore_containers_list(self):
+        return self.pcs_resource_list + self.sidecar_container_list
+
     def create_node(self, name, ssh_client, **kwargs):
         return TripleoTopologyNode(topology=self,
                                    name=name,
                                    ssh_client=ssh_client,
                                    **kwargs)
 
+    def assert_containers_running(self, expected_containers,
+                                  group=None,
+                                  full_name=True, bool_check=False,
+                                  nodenames=None):
+        group = group or 'overcloud'
+        return containers.assert_containers_running(
+            group=group,
+            expected_containers=expected_containers,
+            full_name=full_name,
+            bool_check=bool_check,
+            nodenames=nodenames)
+
+    def list_containers_df(self, group=None):
+        return containers.list_containers_df(group)
+
     def discover_nodes(self):
         self.discover_ssh_proxy_jump_node()
         self.discover_undercloud_nodes()
@@ -237,32 +273,11 @@ def setup_tripleo_topology():
         topology.set_default_openstack_topology_class(TripleoTopology)
 
 
-def get_ip_to_nodes_dict(openstack_nodes=None):
-    if not openstack_nodes:
-        openstack_nodes = topology.list_openstack_nodes(group='overcloud')
-    ip_to_nodes_dict = {str(node.public_ip): node.name for node in
-                        openstack_nodes}
-    return ip_to_nodes_dict
-
-
 def str_is_not_ip(check_str):
     letters = re.compile('[A-Za-z]')
     return bool(letters.match(check_str))
 
 
-def ip_to_hostname(oc_ip):
-    ip_to_nodes_dict = get_ip_to_nodes_dict()
-    oc_ipv6 = oc_ip.replace(".", ":")
-    if netaddr.valid_ipv4(oc_ip) or netaddr.valid_ipv6(oc_ip):
-        return ip_to_nodes_dict[oc_ip]
-    elif netaddr.valid_ipv6(oc_ipv6):
-        LOG.debug("The provided string was a modified IPv6 address: %s",
-                  oc_ip)
-        return ip_to_nodes_dict[oc_ipv6]
-    else:
-        tobiko.fail("wrong IP value provided %s" % oc_ip)
-
-
 def actual_node_groups(groups):
     """return only existing node groups"""
     return set(groups).intersection(topology.list_openstack_node_groups())

tobiko/tripleo/containers.py

@@ -1,6 +1,5 @@
 from __future__ import absolute_import
 
-import abc
 import functools
 import os
 import re
@@ -13,86 +12,21 @@ import pandas
 import tobiko
 from tobiko import config
 from tobiko import podman
-from tobiko import docker
 from tobiko.openstack import neutron
 from tobiko.openstack import topology
+from tobiko import rhosp as rhosp_topology
+from tobiko.rhosp import containers as rhosp_containers
 from tobiko.shell import sh
-from tobiko.shell import ssh
 from tobiko.tripleo import overcloud
-from tobiko.tripleo import topology as tripleo_topology
 
 
 CONF = config.CONF
 LOG = log.getLogger(__name__)
 
 
-class ContainerRuntime(abc.ABC):
-    runtime_name: str
-    version_pattern: typing.Pattern
-
-    def match_version(self, version: str) -> bool:
-        for version_line in version.splitlines():
-            if self.version_pattern.match(version_line) is not None:
-                return True
-        return False
-
-    def get_client(self, ssh_client):
-        for attempt in tobiko.retry(timeout=60.0,
-                                    interval=5.0):
-            try:
-                client = self._get_client(ssh_client=ssh_client)
-                break
-            # TODO chose a better exception type
-            except Exception:
-                if attempt.is_last:
-                    raise
-                LOG.debug('Unable to connect to docker server',
-                          exc_info=1)
-                ssh.reset_default_ssh_port_forward_manager()
-        else:
-            raise RuntimeError("Broken retry loop")
-        return client
-
-    def _get_client(self, ssh_client):
-        raise NotImplementedError
-
-    def list_containers(self, ssh_client):
-        raise NotImplementedError
-
-
-class DockerContainerRuntime(ContainerRuntime):
-    runtime_name = 'docker'
-    version_pattern = re.compile('Docker version .*', re.IGNORECASE)
-
-    def _get_client(self, ssh_client):
-        return docker.get_docker_client(ssh_client=ssh_client,
-                                        sudo=True).connect()
-
-    def list_containers(self, ssh_client):
-        client = self.get_client(ssh_client=ssh_client)
-        return docker.list_docker_containers(client=client)
-
-
-class PodmanContainerRuntime(ContainerRuntime):
-    runtime_name = 'podman'
-    version_pattern = re.compile('Podman version .*', re.IGNORECASE)
-
-    def _get_client(self, ssh_client):
-        return podman.get_podman_client(ssh_client=ssh_client).connect()
-
-    def list_containers(self, ssh_client):
-        client = self.get_client(ssh_client=ssh_client)
-        return podman.list_podman_containers(client=client)
-
-
-DOCKER_RUNTIME = DockerContainerRuntime()
-PODMAN_RUNTIME = PodmanContainerRuntime()
-CONTAINER_RUNTIMES = [PODMAN_RUNTIME, DOCKER_RUNTIME]
-
-
 class ContainerRuntimeFixture(tobiko.SharedFixture):
 
-    runtime: typing.Optional[ContainerRuntime] = None
+    runtime: typing.Optional[rhosp_containers.ContainerRuntime] = None
 
     def setup_fixture(self):
         if overcloud.has_overcloud():
@@ -102,7 +36,7 @@ class ContainerRuntimeFixture(tobiko.SharedFixture):
         self.runtime = None
 
     @staticmethod
-    def get_runtime() -> typing.Optional[ContainerRuntime]:
+    def get_runtime() -> typing.Optional[rhosp_containers.ContainerRuntime]:
         """check what container runtime is running
         and return a handle to it"""
         # TODO THIS LOCKS SSH CLIENT TO CONTROLLER
@@ -112,7 +46,7 @@ class ContainerRuntimeFixture(tobiko.SharedFixture):
                                     ssh_client=node.ssh_client)
             except sh.ShellCommandFailed:
                 continue
-            for runtime in CONTAINER_RUNTIMES:
+            for runtime in rhosp_containers.CONTAINER_RUNTIMES:
                 for version in [result.stdout, result.stderr]:
                     if runtime.match_version(version):
                         return runtime
@@ -121,7 +55,7 @@ class ContainerRuntimeFixture(tobiko.SharedFixture):
             "controller node")
 
 
-def get_container_runtime() -> ContainerRuntime:
+def get_container_runtime() -> rhosp_containers.ContainerRuntime:
     runtime = tobiko.setup_fixture(ContainerRuntimeFixture).runtime
     return runtime
 
@@ -200,10 +134,6 @@ def save_containers_state_to_file(expected_containers_list,):
     return expected_containers_file
 
 
-class ContainerMismatchException(tobiko.TobikoException):
-    pass
-
-
 def assert_containers_running(group, expected_containers, full_name=True,
                               bool_check=False, nodenames=None):
 
@@ -263,7 +193,7 @@ def assert_containers_running(group, expected_containers, full_name=True,
     if not bool_check and failures:
         tobiko.fail(
             'container states mismatched:\n{}'.format('\n'.join(failures)),
-            ContainerMismatchException)
+            rhosp_containers.ContainerMismatchException)
 
     elif bool_check and failures:
         return False
@@ -469,22 +399,22 @@ def comparable_container_keys(container, include_container_objects=False):
     # Differenciate between podman_ver3 with podman-py from earlier api
     if is_podman():
         if podman.Podman_Version_3():
-            con_host_name_stat_obj_tuple = (tripleo_topology.ip_to_hostname(
+            con_host_name_stat_obj_tuple = (rhosp_topology.ip_to_hostname(
                 container.client.base_url.netloc.rsplit('_')[1]),
                                             container.attrs[
                 'Names'][0], container.attrs['State'], container)
 
-            con_host_name_stat_tuple = (tripleo_topology.ip_to_hostname(
+            con_host_name_stat_tuple = (rhosp_topology.ip_to_hostname(
                 container.client.base_url.netloc.rsplit('_')[1]),
                                         container.attrs[
                 'Names'][0], container.attrs['State'])
         else:
 
-            con_host_name_stat_obj_tuple = (tripleo_topology.ip_to_hostname(
+            con_host_name_stat_obj_tuple = (rhosp_topology.ip_to_hostname(
                 container._client._context.hostname),  # pylint: disable=W0212
                 container.data['names'], container.data['status'], container)
 
-            con_host_name_stat_tuple = (tripleo_topology.ip_to_hostname(
+            con_host_name_stat_tuple = (rhosp_topology.ip_to_hostname(
                 container._client._context.hostname),  # pylint: disable=W0212
                 container.data['names'], container.data['status'])
 
@@ -584,54 +514,6 @@ def get_container_states_list(containers_list,
     return container_states_list
 
 
-pcs_resource_list = ['haproxy', 'galera', 'redis', 'ovn-dbs', 'cinder',
-                     'rabbitmq', 'manila', 'ceph', 'pacemaker']
-
-
-sidecar_container_list = [
-    'neutron-haproxy-ovnmeta', 'neutron-haproxy-qrouter',
-    'neutron-dnsmasq-qdhcp', 'neutron-keepalived-qrouter', 'neutron-radvd']
-
-
-def remove_containers_from_comparison(comparable_containers_df):
-    """remove any containers if comparing them with previous status is not
-    necessary or makes no sense
-    """
-    ignore_containers_list = pcs_resource_list + sidecar_container_list
-    for row in comparable_containers_df.iterrows():
-        for ignore_container in ignore_containers_list:
-            if ignore_container in str(row):
-                LOG.info(f'container {ignore_container} has changed state, '
-                         'but that\'s ok - it will be ignored and the test '
-                         f'will not fail due to this: {str(row)}')
-                # if a pcs resource is found , we drop that row
-                comparable_containers_df.drop(row[0], inplace=True)
-                # this row was already dropped, go to next row
-                break
-
-
-def dataframe_difference(df1, df2, which=None):
-    """Find rows which are different between two DataFrames."""
-    comparison_df = df1.merge(df2,
-                              indicator='same_state',
-                              how='outer')
-    # return only non identical rows
-    if which is None:
-        diff_df = comparison_df[comparison_df['same_state'] != 'both']
-
-    else:
-        diff_df = comparison_df[comparison_df['same_state'] == which]
-
-    # if the list of different state containers includes sidecar containers,
-    # ignore them because the existence of these containers depends on the
-    # created resources
-    # if the list of different state containers includes pacemaker resources,
-    # ignore them since the sanity and fault tests check pacemaker status too
-    remove_containers_from_comparison(diff_df)
-
-    return diff_df
-
-
 def assert_equal_containers_state(expected_containers_list=None,
                                   timeout=120, interval=2,
                                   recreate_expected=False):
@@ -676,7 +558,8 @@ def assert_equal_containers_state(expected_containers_list=None,
 
         # execute a `dataframe` diff between the expected and actual containers
         expected_containers_state_changed = \
-            dataframe_difference(expected_containers_list_df,
+            rhosp_containers.dataframe_difference(
+                    expected_containers_list_df,
                     actual_containers_list_df)
         # check for changed state containerstopology
         if not expected_containers_state_changed.empty: