From b957f4a9c8c116c66d023b2ee483602b1276a9b7 Mon Sep 17 00:00:00 2001 From: pinikomarov Date: Sun, 9 Feb 2020 13:21:52 +0200 Subject: [PATCH] add tripleo containers health checks and use it in compute reboot test done refactoring, unit tests are pasing , system tests are passing related bz: https://bugzilla.redhat.com/show_bug.cgi?id=1797892 Change-Id: I2816d03c26e8eeca92c99116e28be52f98aa5141 --- tobiko/docker/_client.py | 2 +- tobiko/openstack/topology/_topology.py | 10 ++ tobiko/podman/_client.py | 74 +++++----- tobiko/tests/faults/ha/test_cloud_recovery.py | 12 +- tobiko/tripleo/containers.py | 136 ++++++++++++++++++ 5 files changed, 198 insertions(+), 36 deletions(-) create mode 100644 tobiko/tripleo/containers.py diff --git a/tobiko/docker/_client.py b/tobiko/docker/_client.py index 62a486b7f..8b2182de6 100644 --- a/tobiko/docker/_client.py +++ b/tobiko/docker/_client.py @@ -30,7 +30,7 @@ def get_docker_client(base_urls=None, ssh_client=None): def list_docker_containers(client=None, **kwargs): try: - containers = docker_client(client).containers.list(**kwargs) + containers = docker_client(client).containers.list(all=True, **kwargs) except _exception.DockerUrlNotFoundError: return tobiko.Selection() else: diff --git a/tobiko/openstack/topology/_topology.py b/tobiko/openstack/topology/_topology.py index 40683ba59..66c6f5258 100644 --- a/tobiko/openstack/topology/_topology.py +++ b/tobiko/openstack/topology/_topology.py @@ -26,6 +26,7 @@ from six.moves.urllib import parse import tobiko from tobiko import docker +from tobiko import podman from tobiko.shell import ip from tobiko.shell import ping from tobiko.shell import sh @@ -97,6 +98,7 @@ def set_default_openstack_topology_class(topology_class): class OpenStackTopologyNode(object): _docker_client = None + _podman_client = None def __init__(self, topology, name, public_ip, ssh_client): self._topology = weakref.ref(topology) @@ -124,6 +126,14 @@ class OpenStackTopologyNode(object): ssh_client=self.ssh_client) return docker_client + @property + def podman_client(self): + podman_client = self._podman_client + if not podman_client: + self._podman_client = podman_client = podman.get_podman_client( + ssh_client=self.ssh_client) + return podman_client + def __repr__(self): return "{cls!s}".format(cls=type(self).__name__, name=self.name) diff --git a/tobiko/podman/_client.py b/tobiko/podman/_client.py index e141d38d0..8fee43eba 100644 --- a/tobiko/podman/_client.py +++ b/tobiko/podman/_client.py @@ -19,6 +19,7 @@ import six import podman + import tobiko from tobiko.podman import _exception from tobiko.podman import _shell @@ -76,28 +77,31 @@ class PodmanClientFixture(tobiko.SharedFixture): def setup_client(self): # setup podman access via varlink - podman_client_setup_cmds = [ - "sudo groupadd -f podman", - "sudo usermod -a -G podman heat-admin", - "sudo chmod o+w /etc/tmpfiles.d", - "sudo echo 'd /run/podman 0750 root heat-admin' > " - "/etc/tmpfiles.d/podman.conf", - "sudo cp /lib/systemd/system/io.podman.socket /etc/systemd/system/" - "io.podman.socket", - "sudo crudini --set /etc/systemd/system/io.podman.socket Socket " - "SocketMode 0660", - "sudo crudini --set /etc/systemd/system/io.podman.socket Socket" - " SocketGroup podman", - "sudo systemctl daemon-reload", - "sudo systemd-tmpfiles --create", - "sudo systemctl enable --now io.podman.socket", - "sudo chown -R root: /run/podman", - "sudo chmod g+rw /run/podman/io.podman", - "sudo systemctl start io.podman.socket" - ] + podman_client_setup_cmds = \ + "sudo test -f /var/varlink_client_access_setup || \ + (sudo groupadd -f podman && \ + sudo usermod -a -G podman heat-admin && \ + sudo chmod -R o=wxr /etc/tmpfiles.d && \ + sudo echo 'd /run/podman 0770 root heat-admin' > \ + /etc/tmpfiles.d/podman.conf && \ + sudo cp /lib/systemd/system/io.podman.socket \ + /etc/systemd/system/io.podman.socket && \ + sudo crudini --set /etc/systemd/system/io.podman.socket Socket \ + SocketMode 0660 && \ + sudo crudini --set /etc/systemd/system/io.podman.socket Socket \ + SocketGroup podman && \ + sudo systemctl daemon-reload && \ + sudo systemd-tmpfiles --create && \ + sudo systemctl enable --now io.podman.socket && \ + sudo chmod 777 /run/podman && \ + sudo chown -R root: /run/podman && \ + sudo chmod g+rw /run/podman/io.podman && \ + sudo chmod 777 /run/podman/io.podman && \ + sudo setenforce 0 && \ + sudo systemctl start io.podman.socket && \ + sudo touch /var/varlink_client_access_setup)" - for cmd in podman_client_setup_cmds: - sh.execute(cmd, ssh_client=self.ssh_client) + sh.execute(podman_client_setup_cmds, ssh_client=self.ssh_client) client = self.client if client is None: @@ -105,19 +109,25 @@ class PodmanClientFixture(tobiko.SharedFixture): return client def create_client(self): - podman_remote_socket = self.discover_podman_socket() - podman_remote_socket_uri = 'unix:/tmp/podman.sock' + for _ in range(360): - remote_uri = 'ssh://{username}@{host}{socket}'.format( - username=self.ssh_client.connect_parameters['username'], - host=self.ssh_client.connect_parameters["hostname"], - socket=podman_remote_socket) + try: + podman_remote_socket = self.discover_podman_socket() + podman_remote_socket_uri = 'unix:/tmp/podman.sock' - client = podman.Client(uri=podman_remote_socket_uri, - remote_uri=remote_uri, - identity_file='~/.ssh/id_rsa') - client.system.ping() - return client + remote_uri = 'ssh://{username}@{host}{socket}'.format( + username=self.ssh_client.connect_parameters['username'], + host=self.ssh_client.connect_parameters["hostname"], + socket=podman_remote_socket) + + client = podman.Client(uri=podman_remote_socket_uri, + remote_uri=remote_uri, + identity_file='~/.ssh/id_rsa') + client.system.ping() + return client + except (ConnectionRefusedError, ConnectionResetError): + # retry + self.create_client() def connect(self): return tobiko.setup_fixture(self).client diff --git a/tobiko/tests/faults/ha/test_cloud_recovery.py b/tobiko/tests/faults/ha/test_cloud_recovery.py index e217368fd..851fde6c1 100644 --- a/tobiko/tests/faults/ha/test_cloud_recovery.py +++ b/tobiko/tests/faults/ha/test_cloud_recovery.py @@ -6,6 +6,7 @@ from tobiko.shell import sh from tobiko.tests.faults.ha import cloud_disruptions from tobiko.tripleo import pacemaker from tobiko.tripleo import processes +from tobiko.tripleo import containers from tobiko.openstack import stacks import tobiko @@ -17,13 +18,12 @@ def nodes_health_check(): # TODO: # Test existing created servers - # ServerStackResourcesTest().test_server_create() # check vm create with ssh and ping checks def check_vm_create(stack_name): - '''stack_name: unique stack name , - so that each time a new vm is created''' + """stack_name: unique stack name , + so that each time a new vm is created""" # create a vm stack = stacks.CirrosServerStackFixture( stack_name=stack_name) @@ -61,8 +61,14 @@ class RebootNodesTest(testtools.TestCase): def test_reboot_computes_recovery(self): nodes_health_check() + computes_containers_dict_before = \ + containers.list_containers(group='compute') cloud_disruptions.reset_all_compute_nodes(hard_reset=True) nodes_health_check() + computes_containers_dict_after = \ + containers.list_containers(group='compute') + containers.assert_equal_containers_state( + computes_containers_dict_before, computes_containers_dict_after) check_vm_create(stack_name=self.id()) # [..] diff --git a/tobiko/tripleo/containers.py b/tobiko/tripleo/containers.py new file mode 100644 index 000000000..0c0806357 --- /dev/null +++ b/tobiko/tripleo/containers.py @@ -0,0 +1,136 @@ +from __future__ import absolute_import + +from oslo_log import log +import pandas + +import tobiko +from tobiko import podman +from tobiko import docker +from tobiko.openstack import topology + + +LOG = log.getLogger(__name__) + + +def container_runtime(): + """check what container runtime is running + and return a handle to it""" + + ssh_client = topology.list_openstack_nodes(group='controller')[ + 0].ssh_client + if docker.is_docker_running(ssh_client=ssh_client): + return docker + else: + return podman + + +container_runtime_type = container_runtime() + + +def list_node_containers(client=None): + """returns a list of containers and their run state""" + + if container_runtime_type == podman: + return container_runtime_type.list_podman_containers(client=client) + + elif container_runtime_type == docker: + return container_runtime_type.list_docker_containers(client=client) + + +def get_container_client(ssh_client=None): + """returns a list of containers and their run state""" + + if container_runtime_type == podman: + return container_runtime_type.get_podman_client( + ssh_client=ssh_client).connect() + + elif container_runtime_type == docker: + return container_runtime_type.get_docker_client( + ssh_client=ssh_client).connect() + + +def list_containers(group=None): + """get list of containers in running state + from specified node group + returns : a list of overcloud_node's running containers""" + + # moved here from topology + # reason : Workaround for : + # AttributeError: module 'tobiko.openstack.topology' has no + # attribute 'container_runtime' + + containers_list = tobiko.Selection() + openstack_nodes = topology.list_openstack_nodes(group=group) + + for node in openstack_nodes: + ssh_client = node.ssh_client + container_client = get_container_client(ssh_client) + node_containers_list = list_node_containers(client=container_client) + containers_list.extend(node_containers_list) + return containers_list + + +def comparable_container_keys(container): + """returns the tuple : 'container_host','container_name', + 'container_state' + """ + if container_runtime_type == podman: + return (container._client._context.hostname, # pylint: disable=W0212 + container.data['names'], container.data['status']) + + elif container_runtime_type == docker: + return (container.attrs['Config']['Hostname'], + container.attrs['State']['Status'], container.attrs['Name']) + + +def get_container_states_list(containers_list): + container_states_list = tobiko.Selection() + container_states_list.extend([comparable_container_keys(container) for + container in containers_list]) + return container_states_list + + +def dataframe_difference(df1, df2, which=None): + """Find rows which are different between two DataFrames.""" + comparison_df = df1.merge(df2, + indicator='same_state', + how='outer') + if which is None: + diff_df = comparison_df[comparison_df['same_state'] != 'both'] + else: + diff_df = comparison_df[comparison_df['same_state'] == which] + return diff_df + + +def assert_equal_containers_state(expected_containers_list, + actual_containers_list): + + """compare container with states from two lists""" + + failures = [] + expected_containers_list_df = pandas.DataFrame( + get_container_states_list(expected_containers_list), + columns=['container_host', 'container_name', 'container_state']) + actual_containers_list_df = pandas.DataFrame( + get_container_states_list(actual_containers_list), + columns=['container_host', 'container_name', 'container_state']) + + LOG.info('expected_containers_list_df: {} '.format( + expected_containers_list_df.to_string(index=False))) + LOG.info('actual_containers_list_df: {} '.format( + actual_containers_list_df.to_string(index=False))) + + # execute a dataframe diff between the excpected and actual containers + expected_containers_state_changed = \ + dataframe_difference(expected_containers_list_df, + actual_containers_list_df) + # check for changed state containers + if not expected_containers_state_changed.empty: + failures.append('expected containers changed state ! : \n\n{}'.format( + expected_containers_state_changed.to_string(index=False))) + + if failures: + tobiko.fail('container states mismatched:\n{!s}', '\n'.join(failures)) + else: + LOG.info("assert_equal_containers_state :" + " OK, all containers are on the same state")