[Podified] Make Topology class to decide about container runtime

Previously it was only used in the Tripleo Topology class and containers
module decided about runtime (docker or podman) which should be used.
Now we have also Podified deployments and we can't rely on checks like
"has_overcloud()" to decide about it.

This patch moves some containers related things to the rhosp and/or
podified modules and let topology class to decide what containers
runtime should be used.

Related-Jira: #OSP-22166
Change-Id: I7187de8884e44eaefcdda6fae160e89242761f39
This commit is contained in:
Slawek Kaplonski 2024-01-12 12:13:48 +01:00
parent 1d845ed565
commit 7d08603c1d
12 changed files with 657 additions and 229 deletions

View File

@ -63,3 +63,5 @@ set_default_openstack_topology_class = (
_topology.set_default_openstack_topology_class) _topology.set_default_openstack_topology_class)
verify_osp_version = _topology.verify_osp_version verify_osp_version = _topology.verify_osp_version
get_config_setting = _topology.get_config_setting get_config_setting = _topology.get_config_setting
node_name_from_hostname = _topology.node_name_from_hostname
remove_duplications = _topology.remove_duplications

View File

@ -311,6 +311,8 @@ class OpenStackTopology(tobiko.SharedFixture):
has_containers = False has_containers = False
container_runtime_cmd = 'docker'
config_file_mappings = { config_file_mappings = {
'ml2_conf.ini': '/etc/neutron/plugins/ml2/ml2_conf.ini', 'ml2_conf.ini': '/etc/neutron/plugins/ml2/ml2_conf.ini',
'bgp-agent.conf': '/etc/ovn-bgp-agent/bgp-agent.conf' 'bgp-agent.conf': '/etc/ovn-bgp-agent/bgp-agent.conf'
@ -396,6 +398,15 @@ class OpenStackTopology(tobiko.SharedFixture):
ssh_client=node.ssh_client) ssh_client=node.ssh_client)
return digger return digger
def assert_containers_running(self, expected_containers,
group=None,
full_name=True, bool_check=False,
nodenames=None):
pass
def list_containers_df(self, group=None):
pass
def discover_nodes(self): def discover_nodes(self):
self.discover_ssh_proxy_jump_node() self.discover_ssh_proxy_jump_node()
self.discover_configured_nodes() self.discover_configured_nodes()

View File

@ -22,6 +22,7 @@ from tobiko.openstack import neutron
from tobiko.openstack import topology from tobiko.openstack import topology
from tobiko.podified import _edpm from tobiko.podified import _edpm
from tobiko.podified import _openshift from tobiko.podified import _openshift
from tobiko.podified import containers
from tobiko import rhosp from tobiko import rhosp
from tobiko.shell import ssh from tobiko.shell import ssh
@ -67,9 +68,29 @@ class PodifiedTopology(rhosp.RhospTopology):
neutron.FRR: 'frr' neutron.FRR: 'frr'
} }
def __init__(self): sidecar_container_list = [
super(PodifiedTopology, self).__init__() 'neutron-haproxy-ovnmeta',
self.ocp_workers = {} 'neutron-dnsmasq-qdhcp'
]
@property
def ignore_containers_list(self):
return self.sidecar_container_list
def assert_containers_running(self, expected_containers,
group=None,
full_name=True, bool_check=False,
nodenames=None):
group = group or ALL_COMPUTES_GROUP_NAME
return containers.assert_containers_running(
group=group,
expected_containers=expected_containers,
full_name=full_name,
bool_check=bool_check,
nodenames=nodenames)
def list_containers_df(self, group=None):
return containers.list_containers_df(group)
def add_node(self, def add_node(self,
hostname: typing.Optional[str] = None, hostname: typing.Optional[str] = None,

View File

@ -0,0 +1,351 @@
from __future__ import absolute_import
import functools
import os
import time
import typing
from oslo_log import log
import pandas
import tobiko
from tobiko import config
from tobiko.openstack import neutron
from tobiko.openstack import topology
from tobiko import rhosp as rhosp_topology
from tobiko.rhosp import containers as rhosp_containers
CONF = config.CONF
LOG = log.getLogger(__name__)
class ContainerRuntimeFixture(tobiko.SharedFixture):
runtime: typing.Optional[rhosp_containers.ContainerRuntime] = None
def setup_fixture(self):
self.runtime = self.get_runtime()
def cleanup_fixture(self):
self.runtime = None
@staticmethod
def get_runtime() -> typing.Optional[rhosp_containers.ContainerRuntime]:
"""return handle to the container runtime"""
return rhosp_containers.PODMAN_RUNTIME
def get_container_runtime() -> rhosp_containers.ContainerRuntime:
runtime = tobiko.setup_fixture(ContainerRuntimeFixture).runtime
return runtime
def get_container_runtime_name() -> str:
return get_container_runtime().runtime_name
def is_docker() -> bool:
return False
def is_podman() -> bool:
return True
def has_container_runtime() -> bool:
return True
@functools.lru_cache()
def list_node_containers(ssh_client):
"""returns a list of containers and their run state"""
return get_container_runtime().list_containers(ssh_client=ssh_client)
def get_container_client(ssh_client=None):
"""returns a list of containers and their run state"""
return get_container_runtime().get_client(ssh_client=ssh_client)
def list_containers_df(group=None):
actual_containers_list = list_containers(group)
return pandas.DataFrame(
get_container_states_list(actual_containers_list),
columns=['container_host', 'container_name', 'container_state'])
def list_containers(group=None):
"""get list of containers in running state
from specified node group
returns : a list of overcloud_node's running containers"""
# moved here from topology
# reason : Workaround for :
# AttributeError: module 'tobiko.openstack.topology' has no
# attribute 'container_runtime'
if group is None:
group = 'compute'
containers_list = tobiko.Selection()
openstack_nodes = topology.list_openstack_nodes(group=group)
for node in openstack_nodes:
LOG.debug(f"List containers for node {node.name}")
node_containers_list = list_node_containers(ssh_client=node.ssh_client)
containers_list.extend(node_containers_list)
return containers_list
expected_containers_file = os.path.expanduser(
'~/expected_containers_list_df.csv')
def save_containers_state_to_file(expected_containers_list,):
expected_containers_list_df = pandas.DataFrame(
get_container_states_list(expected_containers_list),
columns=['container_host', 'container_name', 'container_state'])
expected_containers_list_df.to_csv(
expected_containers_file)
return expected_containers_file
def assert_containers_running(group, expected_containers, full_name=True,
bool_check=False, nodenames=None):
"""assert that all containers specified in the list are running
on the specified openstack group(controller or compute etc..)
if bool_check is True then return only True or false without failing"""
if is_docker():
LOG.info('not checking common containers since we are on docker')
return
failures = []
openstack_nodes = topology.list_openstack_nodes(group=group,
hostnames=nodenames)
for node in openstack_nodes:
node_containers = list_node_containers(ssh_client=node.ssh_client)
containers_list_df = pandas.DataFrame(
get_container_states_list(node_containers),
columns=['container_host', 'container_name', 'container_state'])
# check that the containers are present
LOG.info('node: {} containers list : {}'.format(
node.name, containers_list_df.to_string(index=False)))
for container in expected_containers:
# get container attrs dataframe
if full_name:
container_attrs = containers_list_df.query(
'container_name == "{}"'.format(container))
else:
container_attrs = containers_list_df[
containers_list_df['container_name'].
str.contains(container)]
# check if the container exists
LOG.info('checking container: {}'.format(container))
if container_attrs.empty:
failures.append(
'expected container {} not found on node {} ! : \n\n'.
format(container, node.name))
# if container exists, check it is running
else:
# only one running container is expected
container_running_attrs = container_attrs.query(
'container_state=="running"')
if container_running_attrs.empty:
failures.append(
'expected container {} is not running on node {} , '
'its state is {}! : \n\n'.format(
container, node.name,
container_attrs.container_state.values.item()))
elif len(container_running_attrs) > 1:
failures.append(
'only one running container {} was expected on '
'node {}, but got {}! : \n\n'.format(
container, node.name,
len(container_running_attrs)))
if not bool_check and failures:
tobiko.fail(
'container states mismatched:\n{}'.format('\n'.join(failures)),
rhosp_containers.ContainerMismatchException)
elif bool_check and failures:
return False
else:
LOG.info('All specified containers are in running state! ')
return True
def assert_ovn_containers_running():
if not neutron.has_ovn():
LOG.info("Networking OVN not configured")
return
ovn_containers = ['ovn_metadata_agent',
'ovn_controller']
groups = ['edpm-compute', 'edpm-networker']
for group in groups:
assert_containers_running(group, ovn_containers, full_name=False)
LOG.info("Networking OVN containers verified in running state")
def get_container_states_list(containers_list,
include_container_objects=False):
container_states_list = tobiko.Selection()
container_states_list.extend([comparable_container_keys(
container, include_container_objects=include_container_objects) for
container in containers_list])
return container_states_list
def comparable_container_keys(container, include_container_objects=False):
"""returns the tuple : 'container_host','container_name',
'container_state, container object if specified'
"""
# Differenciate between podman_ver3 with podman-py from earlier api
if include_container_objects:
return (rhosp_topology.ip_to_hostname(
container.client.base_url.netloc.rsplit('_')[1]),
container.attrs['Names'][0], container.attrs['State'],
container)
else:
return (rhosp_topology.ip_to_hostname(
container.client.base_url.netloc.rsplit('_')[1]),
container.attrs['Names'][0],
container.attrs['State'])
@functools.lru_cache()
def list_containers_objects_df():
containers_list = list_containers()
containers_objects_list_df = pandas.DataFrame(
get_container_states_list(
containers_list, include_container_objects=True),
columns=['container_host', 'container_name',
'container_state', 'container_object'])
return containers_objects_list_df
def get_edpm_container(container_name=None, container_host=None,
partial_container_name=None):
"""gets an container object by name on specified host
container"""
con_obj_df = list_containers_objects_df()
if partial_container_name and container_host:
con_obj_df = con_obj_df[con_obj_df['container_name'].str.contains(
partial_container_name)]
contaniner_obj = con_obj_df.query(
'container_host == "{container_host}"'.format(
container_host=container_host))['container_object']
elif container_host:
contaniner_obj = con_obj_df.query(
'container_name == "{container_name}"'
' and container_host == "{container_host}"'.
format(container_host=container_host,
container_name=container_name)).container_object
else:
contaniner_obj = con_obj_df.query(
'container_name == "{container_name}"'.
format(container_name=container_name)).container_object
if not contaniner_obj.empty:
return contaniner_obj.values[0]
else:
tobiko.fail('container {} not found!'.format(container_name))
def action_on_container(action: str,
container_name=None,
container_host=None,
partial_container_name=None):
"""take a container and perform an action on it
actions are as defined in : podman/libs/containers.py:14/164"""
LOG.debug(f"Executing '{action}' action on container "
f"'{container_name}@{container_host}'...")
container = get_edpm_container(
container_name=container_name,
container_host=container_host,
partial_container_name=partial_container_name)
container_class: typing.Type = type(container)
# we get the specified action as function from podman lib
action_method: typing.Optional[typing.Callable] = getattr(
container_class, action, None)
if action_method is None:
raise TypeError(f"Unsupported container action for class :"
f" {container_class}")
if not callable(action_method):
raise TypeError(
f"Attribute '{container_class.__qualname__}.{action}' value "
f" is not a method: {action_method!r}")
LOG.debug(f"Calling '{action_method}' action on container "
f"'{container}'")
return action_method(container)
def assert_equal_containers_state(expected_containers_list=None,
timeout=120, interval=2,
recreate_expected=False):
"""compare all edpm container states with using two lists:
one is current , the other some past list
first time this method runs it creates a file holding overcloud
containers' states: ~/expected_containers_list_df.csv'
second time it creates a current containers states list and
compares them, they must be identical"""
# if we have a file or an explicit variable use that , otherwise create
# and return
if recreate_expected or (not expected_containers_list and
not os.path.exists(expected_containers_file)):
save_containers_state_to_file(list_containers())
return
elif expected_containers_list:
expected_containers_list_df = pandas.DataFrame(
get_container_states_list(expected_containers_list),
columns=['container_host', 'container_name', 'container_state'])
elif os.path.exists(expected_containers_file):
expected_containers_list_df = pandas.read_csv(
expected_containers_file)
failures = []
start = time.time()
error_info = 'Output explanation: left_only is the original state, ' \
'right_only is the new state'
while time.time() - start < timeout:
failures = []
actual_containers_list_df = list_containers_df()
LOG.info('expected_containers_list_df: {} '.format(
expected_containers_list_df.to_string(index=False)))
LOG.info('actual_containers_list_df: {} '.format(
actual_containers_list_df.to_string(index=False)))
# execute a `dataframe` diff between the expected and actual containers
expected_containers_state_changed = \
rhosp_containers.dataframe_difference(
expected_containers_list_df,
actual_containers_list_df)
# check for changed state containerstopology
if not expected_containers_state_changed.empty:
failures.append('expected containers changed state ! : '
'\n\n{}\n{}'.format(
expected_containers_state_changed.
to_string(index=False), error_info))
LOG.info('container states mismatched:\n{}\n'.format(failures))
time.sleep(interval)
# clear cache to obtain new data
list_node_containers.cache_clear()
else:
LOG.info("assert_equal_containers_state :"
" OK, all containers are on the same state")
return
if failures:
tobiko.fail('container states mismatched:\n{!s}', '\n'.join(
failures))

View File

@ -21,3 +21,5 @@ RhospNode = _topology.RhospNode
get_rhosp_release = _version_utils.get_rhosp_release get_rhosp_release = _version_utils.get_rhosp_release
get_rhosp_version = _version_utils.get_rhosp_version get_rhosp_version = _version_utils.get_rhosp_version
ip_to_hostname = _topology.ip_to_hostname

View File

@ -27,6 +27,27 @@ from tobiko.shell import ssh
LOG = log.getLogger(__name__) LOG = log.getLogger(__name__)
def get_ip_to_nodes_dict(group, openstack_nodes=None):
if not openstack_nodes:
openstack_nodes = topology.list_openstack_nodes(group=group)
ip_to_nodes_dict = {str(node.public_ip): node.name for node in
openstack_nodes}
return ip_to_nodes_dict
def ip_to_hostname(oc_ip, group=None):
ip_to_nodes_dict = get_ip_to_nodes_dict(group)
oc_ipv6 = oc_ip.replace(".", ":")
if netaddr.valid_ipv4(oc_ip) or netaddr.valid_ipv6(oc_ip):
return ip_to_nodes_dict[oc_ip]
elif netaddr.valid_ipv6(oc_ipv6):
LOG.debug("The provided string was a modified IPv6 address: %s",
oc_ip)
return ip_to_nodes_dict[oc_ipv6]
else:
tobiko.fail("wrong IP value provided %s" % oc_ip)
class RhospTopology(topology.OpenStackTopology): class RhospTopology(topology.OpenStackTopology):
"""Base topology for Red Hat OpenStack deployments. """Base topology for Red Hat OpenStack deployments.
@ -36,6 +57,11 @@ class RhospTopology(topology.OpenStackTopology):
""" """
has_containers = True has_containers = True
container_runtime_cmd = 'podman'
@property
def ignore_containers_list(self):
return None
class RhospNode(topology.OpenStackTopologyNode): class RhospNode(topology.OpenStackTopologyNode):

123
tobiko/rhosp/containers.py Normal file
View File

@ -0,0 +1,123 @@
from __future__ import absolute_import
import abc
import re
import typing
from oslo_log import log
import tobiko
from tobiko.openstack import topology
from tobiko import podman
from tobiko import docker
from tobiko.shell import ssh
LOG = log.getLogger(__name__)
class ContainerRuntime(abc.ABC):
runtime_name: str
version_pattern: typing.Pattern
def match_version(self, version: str) -> bool:
for version_line in version.splitlines():
if self.version_pattern.match(version_line) is not None:
return True
return False
def get_client(self, ssh_client):
for attempt in tobiko.retry(timeout=60.0,
interval=5.0):
try:
client = self._get_client(ssh_client=ssh_client)
break
# TODO chose a better exception type
except Exception:
if attempt.is_last:
raise
LOG.debug('Unable to connect to docker server',
exc_info=1)
ssh.reset_default_ssh_port_forward_manager()
else:
raise RuntimeError("Broken retry loop")
return client
def _get_client(self, ssh_client):
raise NotImplementedError
def list_containers(self, ssh_client):
raise NotImplementedError
class DockerContainerRuntime(ContainerRuntime):
runtime_name = 'docker'
version_pattern = re.compile('Docker version .*', re.IGNORECASE)
def _get_client(self, ssh_client):
return docker.get_docker_client(ssh_client=ssh_client,
sudo=True).connect()
def list_containers(self, ssh_client):
client = self.get_client(ssh_client=ssh_client)
return docker.list_docker_containers(client=client)
class PodmanContainerRuntime(ContainerRuntime):
runtime_name = 'podman'
version_pattern = re.compile('Podman version .*', re.IGNORECASE)
def _get_client(self, ssh_client):
return podman.get_podman_client(ssh_client=ssh_client).connect()
def list_containers(self, ssh_client):
client = self.get_client(ssh_client=ssh_client)
return podman.list_podman_containers(client=client)
DOCKER_RUNTIME = DockerContainerRuntime()
PODMAN_RUNTIME = PodmanContainerRuntime()
CONTAINER_RUNTIMES = [PODMAN_RUNTIME, DOCKER_RUNTIME]
class ContainerMismatchException(tobiko.TobikoException):
pass
def remove_containers_from_comparison(comparable_containers_df):
"""remove any containers if comparing them with previous status is not
necessary or makes no sense
"""
os_topology = topology.get_openstack_topology()
for row in comparable_containers_df.iterrows():
for ignore_container in os_topology.ignore_containers_list:
if ignore_container in str(row):
LOG.info(f'container {ignore_container} has changed state, '
'but that\'s ok - it will be ignored and the test '
f'will not fail due to this: {str(row)}')
# if a pcs resource is found , we drop that row
comparable_containers_df.drop(row[0], inplace=True)
# this row was already dropped, go to next row
break
def dataframe_difference(df1, df2, which=None):
"""Find rows which are different between two DataFrames."""
comparison_df = df1.merge(df2,
indicator='same_state',
how='outer')
# return only non identical rows
if which is None:
diff_df = comparison_df[comparison_df['same_state'] != 'both']
else:
diff_df = comparison_df[comparison_df['same_state'] == which]
# if the list of different state containers includes sidecar containers,
# ignore them because the existence of these containers depends on the
# created resources
# if the list of different state containers includes pacemaker resources,
# ignore them since the sanity and fault tests check pacemaker status too
remove_containers_from_comparison(diff_df)
return diff_df

View File

@ -31,8 +31,6 @@ from tobiko.openstack import tests
from tobiko.openstack import topology from tobiko.openstack import topology
from tobiko.shell import ping from tobiko.shell import ping
from tobiko.shell import sh from tobiko.shell import sh
from tobiko.tripleo import containers
from tobiko.tripleo import overcloud
LOG = log.getLogger(__name__) LOG = log.getLogger(__name__)
@ -69,10 +67,7 @@ class BaseAgentTest(testtools.TestCase):
@property @property
def container_runtime_name(self): def container_runtime_name(self):
if overcloud.has_overcloud(): return topology.get_openstack_topology().container_runtime_cmd
return containers.get_container_runtime_name()
else:
return 'docker'
def get_agent_container_name(self, agent_name): def get_agent_container_name(self, agent_name):
try: try:
@ -88,8 +83,9 @@ class BaseAgentTest(testtools.TestCase):
self.agent_name) self.agent_name)
if not self.container_name: if not self.container_name:
return return
oc_containers_df = containers.list_containers_df().query( oc_containers_df = \
f'container_name == "{self.container_name}"') topology.get_openstack_topology().list_containers_df().query(
f'container_name == "{self.container_name}"')
LOG.debug( LOG.debug(
f"{self.container_name} container found:\n{oc_containers_df}") f"{self.container_name} container found:\n{oc_containers_df}")
@ -166,8 +162,7 @@ class BaseAgentTest(testtools.TestCase):
''' '''
self._do_agent_action(START, hosts) self._do_agent_action(START, hosts)
if self.container_name: if self.container_name:
containers.assert_containers_running( topology.get_openstack_topology().assert_containers_running(
group='overcloud',
expected_containers=[self.container_name], expected_containers=[self.container_name],
nodenames=hosts or self.hosts) nodenames=hosts or self.hosts)
@ -181,8 +176,7 @@ class BaseAgentTest(testtools.TestCase):
''' '''
self._do_agent_action(RESTART, hosts) self._do_agent_action(RESTART, hosts)
if self.container_name: if self.container_name:
containers.assert_containers_running( topology.get_openstack_topology().assert_containers_running(
group='overcloud',
expected_containers=[self.container_name], expected_containers=[self.container_name],
nodenames=hosts or self.hosts) nodenames=hosts or self.hosts)
@ -213,8 +207,7 @@ class BaseAgentTest(testtools.TestCase):
sudo=True) sudo=True)
if self.container_name: if self.container_name:
containers.assert_containers_running( topology.get_openstack_topology().assert_containers_running(
group='overcloud',
expected_containers=[self.container_name], expected_containers=[self.container_name],
nodenames=hosts or self.hosts) nodenames=hosts or self.hosts)

View File

@ -15,6 +15,7 @@ from __future__ import absolute_import
import testtools import testtools
from tobiko.rhosp import containers as rhosp_containers
from tobiko.tripleo import containers from tobiko.tripleo import containers
@ -23,7 +24,7 @@ class RuntimeRuntimeTest(testtools.TestCase):
def test_get_container_runtime(self): def test_get_container_runtime(self):
runtime = containers.get_container_runtime() runtime = containers.get_container_runtime()
self.assertIsInstance(runtime, containers.ContainerRuntime) self.assertIsInstance(runtime, rhosp_containers.ContainerRuntime)
def test_list_containers(self): def test_list_containers(self):
containers_list = containers.list_containers() containers_list = containers.list_containers()

View File

@ -26,6 +26,7 @@ import pandas
import tobiko import tobiko
from tobiko.openstack import neutron from tobiko.openstack import neutron
from tobiko.openstack import topology from tobiko.openstack import topology
from tobiko.rhosp import containers as rhosp_containers
from tobiko.shell import sh from tobiko.shell import sh
from tobiko import tripleo from tobiko import tripleo
from tobiko.tripleo import containers from tobiko.tripleo import containers
@ -34,8 +35,15 @@ from tobiko.tripleo import containers
LOG = log.getLogger(__name__) LOG = log.getLogger(__name__)
class BaseContainersHealtTest(testtools.TestCase):
def _assert_containers_running(self, group, expected_containers):
topology.get_openstack_topology().assert_containers_running(
group=group, expected_containers=expected_containers)
@tripleo.skip_if_missing_overcloud @tripleo.skip_if_missing_overcloud
class ContainersHealthTest(testtools.TestCase): class ContainersHealthTest(BaseContainersHealtTest):
# TODO(eolivare): refactor this class, because it replicates some code from # TODO(eolivare): refactor this class, because it replicates some code from
# tobiko/tripleo/containers.py and its tests may be duplicating what # tobiko/tripleo/containers.py and its tests may be duplicating what
# test_0vercloud_health_check already covers when it calls # test_0vercloud_health_check already covers when it calls
@ -49,158 +57,150 @@ class ContainersHealthTest(testtools.TestCase):
def test_cinder_api(self): def test_cinder_api(self):
"""check that all common tripleo containers are running""" """check that all common tripleo containers are running"""
containers.assert_containers_running('controller', ['cinder_api']) self._assert_containers_running('controller', ['cinder_api'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_rsync(self): def test_swift_rsync(self):
containers.assert_containers_running('controller', ['swift_rsync']) self._assert_containers_running('controller', ['swift_rsync'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_proxy(self): def test_swift_proxy(self):
containers.assert_containers_running('controller', ['swift_proxy']) self._assert_containers_running('controller', ['swift_proxy'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_object_updater(self): def test_swift_object_updater(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller', ['swift_object_updater'])
['swift_object_updater'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_object_server(self): def test_swift_object_server(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller', ['swift_object_server'])
['swift_object_server'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_object_replicator(self): def test_swift_object_replicator(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller',
['swift_object_replicator']) ['swift_object_replicator'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_object_expirer(self): def test_swift_object_expirer(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller', ['swift_object_expirer'])
['swift_object_expirer'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_object_auditor(self): def test_swift_object_auditor(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller', ['swift_object_auditor'])
['swift_object_auditor'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_container_updater(self): def test_swift_container_updater(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller',
['swift_container_updater']) ['swift_container_updater'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_container_server(self): def test_swift_container_server(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller',
['swift_container_server']) ['swift_container_server'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_container_replicator(self): def test_swift_container_replicator(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller',
['swift_container_replicator']) ['swift_container_replicator'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_container_auditor(self): def test_swift_container_auditor(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller',
['swift_container_auditor']) ['swift_container_auditor'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_account_server(self): def test_swift_account_server(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller', ['swift_account_server'])
['swift_account_server'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_account_replicator(self): def test_swift_account_replicator(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller',
['swift_account_replicator']) ['swift_account_replicator'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_account_reaper(self): def test_swift_account_reaper(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller', ['swift_account_reaper'])
['swift_account_reaper'])
@tripleo.skip_if_ceph_rgw() @tripleo.skip_if_ceph_rgw()
def test_swift_account_auditor(self): def test_swift_account_auditor(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller',
['swift_account_auditor']) ['swift_account_auditor'])
def test_nova_vnc_proxy(self): def test_nova_vnc_proxy(self):
containers.assert_containers_running('controller', ['nova_vnc_proxy']) self._assert_containers_running('controller', ['nova_vnc_proxy'])
def test_nova_scheduler(self): def test_nova_scheduler(self):
containers.assert_containers_running('controller', ['nova_scheduler']) self._assert_containers_running('controller', ['nova_scheduler'])
def test_nova_metadata(self): def test_nova_metadata(self):
containers.assert_containers_running('controller', ['nova_metadata']) self._assert_containers_running('controller', ['nova_metadata'])
def test_nova_conductor(self): def test_nova_conductor(self):
containers.assert_containers_running('controller', ['nova_conductor']) self._assert_containers_running('controller', ['nova_conductor'])
def test_nova_api_cron(self): def test_nova_api_cron(self):
containers.assert_containers_running('controller', ['nova_api_cron']) self._assert_containers_running('controller', ['nova_api_cron'])
def test_nova_api(self): def test_nova_api(self):
containers.assert_containers_running('controller', ['nova_api']) self._assert_containers_running('controller', ['nova_api'])
def test_neutron_api(self): def test_neutron_api(self):
containers.assert_containers_running('controller', ['neutron_api']) self._assert_containers_running('controller', ['neutron_api'])
def test_memcached(self): def test_memcached(self):
containers.assert_containers_running('controller', ['memcached']) self._assert_containers_running('controller', ['memcached'])
def test_controller_logrotate_crond(self): def test_controller_logrotate_crond(self):
containers.assert_containers_running('controller', ['logrotate_crond']) self._assert_containers_running('controller', ['logrotate_crond'])
def test_keystone(self): def test_keystone(self):
containers.assert_containers_running('controller', ['keystone']) self._assert_containers_running('controller', ['keystone'])
def test_controller_iscsid(self): def test_controller_iscsid(self):
containers.assert_containers_running('controller', ['iscsid']) self._assert_containers_running('controller', ['iscsid'])
def test_horizon(self): def test_horizon(self):
containers.assert_containers_running('controller', ['horizon']) self._assert_containers_running('controller', ['horizon'])
def test_heat_engine(self): def test_heat_engine(self):
containers.assert_containers_running('controller', ['heat_engine']) self._assert_containers_running('controller', ['heat_engine'])
def test_heat_api_cron(self): def test_heat_api_cron(self):
containers.assert_containers_running('controller', ['heat_api_cron']) self._assert_containers_running('controller', ['heat_api_cron'])
def test_heat_api_cfn(self): def test_heat_api_cfn(self):
containers.assert_containers_running('controller', ['heat_api_cfn']) self._assert_containers_running('controller', ['heat_api_cfn'])
def test_heat_api(self): def test_heat_api(self):
containers.assert_containers_running('controller', ['heat_api']) self._assert_containers_running('controller', ['heat_api'])
def test_glance_api(self): def test_glance_api(self):
containers.assert_containers_running('controller', ['glance_api']) self._assert_containers_running('controller', ['glance_api'])
def test_cinder_scheduler(self): def test_cinder_scheduler(self):
containers.assert_containers_running('controller', self._assert_containers_running('controller', ['cinder_scheduler'])
['cinder_scheduler'])
def test_cinder_api_cron(self): def test_cinder_api_cron(self):
containers.assert_containers_running('controller', ['cinder_api_cron']) self._assert_containers_running('controller', ['cinder_api_cron'])
def test_compute_iscsid(self): def test_compute_iscsid(self):
containers.assert_containers_running('compute', ['iscsid']) self._assert_containers_running('compute', ['iscsid'])
def test_compute_logrotate_crond(self): def test_compute_logrotate_crond(self):
containers.assert_containers_running('compute', ['logrotate_crond']) self._assert_containers_running('compute', ['logrotate_crond'])
def test_nova_compute(self): def test_nova_compute(self):
containers.assert_containers_running('compute', ['nova_compute']) self._assert_containers_running('compute', ['nova_compute'])
def test_nova_libvirt(self): def test_nova_libvirt(self):
nova_libvirt = containers.get_libvirt_container_name() nova_libvirt = containers.get_libvirt_container_name()
containers.assert_containers_running('compute', [nova_libvirt]) self._assert_containers_running('compute', [nova_libvirt])
def test_nova_migration_target(self): def test_nova_migration_target(self):
containers.assert_containers_running('compute', self._assert_containers_running('compute', ['nova_migration_target'])
['nova_migration_target'])
def test_nova_virtlogd(self): def test_nova_virtlogd(self):
containers.assert_containers_running('compute', ['nova_virtlogd']) self._assert_containers_running('compute', ['nova_virtlogd'])
def test_ovn_containers_running(self): def test_ovn_containers_running(self):
containers.assert_ovn_containers_running() containers.assert_ovn_containers_running()
@ -244,8 +244,8 @@ class ContainersHealthTest(testtools.TestCase):
# execute a `dataframe` diff between the expected # execute a `dataframe` diff between the expected
# and actual containers # and actual containers
expected_containers_state_changed = \ expected_containers_state_changed = \
containers.dataframe_difference(expected_containers_list_df, rhosp_containers.dataframe_difference(
actual_containers_list_df) expected_containers_list_df, actual_containers_list_df)
# check for changed state containerstopology # check for changed state containerstopology
if not expected_containers_state_changed.empty: if not expected_containers_state_changed.empty:
failures.append('expected containers changed state ! : ' failures.append('expected containers changed state ! : '

View File

@ -31,6 +31,7 @@ from tobiko.shell import sh
from tobiko.shell import ssh from tobiko.shell import ssh
from tobiko.tripleo import _overcloud from tobiko.tripleo import _overcloud
from tobiko.tripleo import _undercloud from tobiko.tripleo import _undercloud
from tobiko.tripleo import containers
CONF = config.CONF CONF = config.CONF
LOG = log.getLogger(__name__) LOG = log.getLogger(__name__)
@ -84,12 +85,47 @@ class TripleoTopology(rhosp.RhospTopology):
neutron.SERVER: '/var/log/containers/neutron/server.log*', neutron.SERVER: '/var/log/containers/neutron/server.log*',
} }
_container_runtime_cmd = None
pcs_resource_list = ['haproxy', 'galera', 'redis', 'ovn-dbs', 'cinder',
'rabbitmq', 'manila', 'ceph', 'pacemaker']
sidecar_container_list = [
'neutron-haproxy-ovnmeta', 'neutron-haproxy-qrouter',
'neutron-dnsmasq-qdhcp', 'neutron-keepalived-qrouter', 'neutron-radvd']
@property
def container_runtime_cmd(self):
if self._container_runtime_cmd is None:
self._container_runtime_cmd = \
containers.get_container_runtime_name()
return self._container_runtime_cmd
@property
def ignore_containers_list(self):
return self.pcs_resource_list + self.sidecar_container_list
def create_node(self, name, ssh_client, **kwargs): def create_node(self, name, ssh_client, **kwargs):
return TripleoTopologyNode(topology=self, return TripleoTopologyNode(topology=self,
name=name, name=name,
ssh_client=ssh_client, ssh_client=ssh_client,
**kwargs) **kwargs)
def assert_containers_running(self, expected_containers,
group=None,
full_name=True, bool_check=False,
nodenames=None):
group = group or 'overcloud'
return containers.assert_containers_running(
group=group,
expected_containers=expected_containers,
full_name=full_name,
bool_check=bool_check,
nodenames=nodenames)
def list_containers_df(self, group=None):
return containers.list_containers_df(group)
def discover_nodes(self): def discover_nodes(self):
self.discover_ssh_proxy_jump_node() self.discover_ssh_proxy_jump_node()
self.discover_undercloud_nodes() self.discover_undercloud_nodes()
@ -237,32 +273,11 @@ def setup_tripleo_topology():
topology.set_default_openstack_topology_class(TripleoTopology) topology.set_default_openstack_topology_class(TripleoTopology)
def get_ip_to_nodes_dict(openstack_nodes=None):
if not openstack_nodes:
openstack_nodes = topology.list_openstack_nodes(group='overcloud')
ip_to_nodes_dict = {str(node.public_ip): node.name for node in
openstack_nodes}
return ip_to_nodes_dict
def str_is_not_ip(check_str): def str_is_not_ip(check_str):
letters = re.compile('[A-Za-z]') letters = re.compile('[A-Za-z]')
return bool(letters.match(check_str)) return bool(letters.match(check_str))
def ip_to_hostname(oc_ip):
ip_to_nodes_dict = get_ip_to_nodes_dict()
oc_ipv6 = oc_ip.replace(".", ":")
if netaddr.valid_ipv4(oc_ip) or netaddr.valid_ipv6(oc_ip):
return ip_to_nodes_dict[oc_ip]
elif netaddr.valid_ipv6(oc_ipv6):
LOG.debug("The provided string was a modified IPv6 address: %s",
oc_ip)
return ip_to_nodes_dict[oc_ipv6]
else:
tobiko.fail("wrong IP value provided %s" % oc_ip)
def actual_node_groups(groups): def actual_node_groups(groups):
"""return only existing node groups""" """return only existing node groups"""
return set(groups).intersection(topology.list_openstack_node_groups()) return set(groups).intersection(topology.list_openstack_node_groups())

View File

@ -1,6 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
import abc
import functools import functools
import os import os
import re import re
@ -13,86 +12,21 @@ import pandas
import tobiko import tobiko
from tobiko import config from tobiko import config
from tobiko import podman from tobiko import podman
from tobiko import docker
from tobiko.openstack import neutron from tobiko.openstack import neutron
from tobiko.openstack import topology from tobiko.openstack import topology
from tobiko import rhosp as rhosp_topology
from tobiko.rhosp import containers as rhosp_containers
from tobiko.shell import sh from tobiko.shell import sh
from tobiko.shell import ssh
from tobiko.tripleo import overcloud from tobiko.tripleo import overcloud
from tobiko.tripleo import topology as tripleo_topology
CONF = config.CONF CONF = config.CONF
LOG = log.getLogger(__name__) LOG = log.getLogger(__name__)
class ContainerRuntime(abc.ABC):
runtime_name: str
version_pattern: typing.Pattern
def match_version(self, version: str) -> bool:
for version_line in version.splitlines():
if self.version_pattern.match(version_line) is not None:
return True
return False
def get_client(self, ssh_client):
for attempt in tobiko.retry(timeout=60.0,
interval=5.0):
try:
client = self._get_client(ssh_client=ssh_client)
break
# TODO chose a better exception type
except Exception:
if attempt.is_last:
raise
LOG.debug('Unable to connect to docker server',
exc_info=1)
ssh.reset_default_ssh_port_forward_manager()
else:
raise RuntimeError("Broken retry loop")
return client
def _get_client(self, ssh_client):
raise NotImplementedError
def list_containers(self, ssh_client):
raise NotImplementedError
class DockerContainerRuntime(ContainerRuntime):
runtime_name = 'docker'
version_pattern = re.compile('Docker version .*', re.IGNORECASE)
def _get_client(self, ssh_client):
return docker.get_docker_client(ssh_client=ssh_client,
sudo=True).connect()
def list_containers(self, ssh_client):
client = self.get_client(ssh_client=ssh_client)
return docker.list_docker_containers(client=client)
class PodmanContainerRuntime(ContainerRuntime):
runtime_name = 'podman'
version_pattern = re.compile('Podman version .*', re.IGNORECASE)
def _get_client(self, ssh_client):
return podman.get_podman_client(ssh_client=ssh_client).connect()
def list_containers(self, ssh_client):
client = self.get_client(ssh_client=ssh_client)
return podman.list_podman_containers(client=client)
DOCKER_RUNTIME = DockerContainerRuntime()
PODMAN_RUNTIME = PodmanContainerRuntime()
CONTAINER_RUNTIMES = [PODMAN_RUNTIME, DOCKER_RUNTIME]
class ContainerRuntimeFixture(tobiko.SharedFixture): class ContainerRuntimeFixture(tobiko.SharedFixture):
runtime: typing.Optional[ContainerRuntime] = None runtime: typing.Optional[rhosp_containers.ContainerRuntime] = None
def setup_fixture(self): def setup_fixture(self):
if overcloud.has_overcloud(): if overcloud.has_overcloud():
@ -102,7 +36,7 @@ class ContainerRuntimeFixture(tobiko.SharedFixture):
self.runtime = None self.runtime = None
@staticmethod @staticmethod
def get_runtime() -> typing.Optional[ContainerRuntime]: def get_runtime() -> typing.Optional[rhosp_containers.ContainerRuntime]:
"""check what container runtime is running """check what container runtime is running
and return a handle to it""" and return a handle to it"""
# TODO THIS LOCKS SSH CLIENT TO CONTROLLER # TODO THIS LOCKS SSH CLIENT TO CONTROLLER
@ -112,7 +46,7 @@ class ContainerRuntimeFixture(tobiko.SharedFixture):
ssh_client=node.ssh_client) ssh_client=node.ssh_client)
except sh.ShellCommandFailed: except sh.ShellCommandFailed:
continue continue
for runtime in CONTAINER_RUNTIMES: for runtime in rhosp_containers.CONTAINER_RUNTIMES:
for version in [result.stdout, result.stderr]: for version in [result.stdout, result.stderr]:
if runtime.match_version(version): if runtime.match_version(version):
return runtime return runtime
@ -121,7 +55,7 @@ class ContainerRuntimeFixture(tobiko.SharedFixture):
"controller node") "controller node")
def get_container_runtime() -> ContainerRuntime: def get_container_runtime() -> rhosp_containers.ContainerRuntime:
runtime = tobiko.setup_fixture(ContainerRuntimeFixture).runtime runtime = tobiko.setup_fixture(ContainerRuntimeFixture).runtime
return runtime return runtime
@ -200,10 +134,6 @@ def save_containers_state_to_file(expected_containers_list,):
return expected_containers_file return expected_containers_file
class ContainerMismatchException(tobiko.TobikoException):
pass
def assert_containers_running(group, expected_containers, full_name=True, def assert_containers_running(group, expected_containers, full_name=True,
bool_check=False, nodenames=None): bool_check=False, nodenames=None):
@ -263,7 +193,7 @@ def assert_containers_running(group, expected_containers, full_name=True,
if not bool_check and failures: if not bool_check and failures:
tobiko.fail( tobiko.fail(
'container states mismatched:\n{}'.format('\n'.join(failures)), 'container states mismatched:\n{}'.format('\n'.join(failures)),
ContainerMismatchException) rhosp_containers.ContainerMismatchException)
elif bool_check and failures: elif bool_check and failures:
return False return False
@ -469,22 +399,22 @@ def comparable_container_keys(container, include_container_objects=False):
# Differenciate between podman_ver3 with podman-py from earlier api # Differenciate between podman_ver3 with podman-py from earlier api
if is_podman(): if is_podman():
if podman.Podman_Version_3(): if podman.Podman_Version_3():
con_host_name_stat_obj_tuple = (tripleo_topology.ip_to_hostname( con_host_name_stat_obj_tuple = (rhosp_topology.ip_to_hostname(
container.client.base_url.netloc.rsplit('_')[1]), container.client.base_url.netloc.rsplit('_')[1]),
container.attrs[ container.attrs[
'Names'][0], container.attrs['State'], container) 'Names'][0], container.attrs['State'], container)
con_host_name_stat_tuple = (tripleo_topology.ip_to_hostname( con_host_name_stat_tuple = (rhosp_topology.ip_to_hostname(
container.client.base_url.netloc.rsplit('_')[1]), container.client.base_url.netloc.rsplit('_')[1]),
container.attrs[ container.attrs[
'Names'][0], container.attrs['State']) 'Names'][0], container.attrs['State'])
else: else:
con_host_name_stat_obj_tuple = (tripleo_topology.ip_to_hostname( con_host_name_stat_obj_tuple = (rhosp_topology.ip_to_hostname(
container._client._context.hostname), # pylint: disable=W0212 container._client._context.hostname), # pylint: disable=W0212
container.data['names'], container.data['status'], container) container.data['names'], container.data['status'], container)
con_host_name_stat_tuple = (tripleo_topology.ip_to_hostname( con_host_name_stat_tuple = (rhosp_topology.ip_to_hostname(
container._client._context.hostname), # pylint: disable=W0212 container._client._context.hostname), # pylint: disable=W0212
container.data['names'], container.data['status']) container.data['names'], container.data['status'])
@ -584,54 +514,6 @@ def get_container_states_list(containers_list,
return container_states_list return container_states_list
pcs_resource_list = ['haproxy', 'galera', 'redis', 'ovn-dbs', 'cinder',
'rabbitmq', 'manila', 'ceph', 'pacemaker']
sidecar_container_list = [
'neutron-haproxy-ovnmeta', 'neutron-haproxy-qrouter',
'neutron-dnsmasq-qdhcp', 'neutron-keepalived-qrouter', 'neutron-radvd']
def remove_containers_from_comparison(comparable_containers_df):
"""remove any containers if comparing them with previous status is not
necessary or makes no sense
"""
ignore_containers_list = pcs_resource_list + sidecar_container_list
for row in comparable_containers_df.iterrows():
for ignore_container in ignore_containers_list:
if ignore_container in str(row):
LOG.info(f'container {ignore_container} has changed state, '
'but that\'s ok - it will be ignored and the test '
f'will not fail due to this: {str(row)}')
# if a pcs resource is found , we drop that row
comparable_containers_df.drop(row[0], inplace=True)
# this row was already dropped, go to next row
break
def dataframe_difference(df1, df2, which=None):
"""Find rows which are different between two DataFrames."""
comparison_df = df1.merge(df2,
indicator='same_state',
how='outer')
# return only non identical rows
if which is None:
diff_df = comparison_df[comparison_df['same_state'] != 'both']
else:
diff_df = comparison_df[comparison_df['same_state'] == which]
# if the list of different state containers includes sidecar containers,
# ignore them because the existence of these containers depends on the
# created resources
# if the list of different state containers includes pacemaker resources,
# ignore them since the sanity and fault tests check pacemaker status too
remove_containers_from_comparison(diff_df)
return diff_df
def assert_equal_containers_state(expected_containers_list=None, def assert_equal_containers_state(expected_containers_list=None,
timeout=120, interval=2, timeout=120, interval=2,
recreate_expected=False): recreate_expected=False):
@ -676,8 +558,9 @@ def assert_equal_containers_state(expected_containers_list=None,
# execute a `dataframe` diff between the expected and actual containers # execute a `dataframe` diff between the expected and actual containers
expected_containers_state_changed = \ expected_containers_state_changed = \
dataframe_difference(expected_containers_list_df, rhosp_containers.dataframe_difference(
actual_containers_list_df) expected_containers_list_df,
actual_containers_list_df)
# check for changed state containerstopology # check for changed state containerstopology
if not expected_containers_state_changed.empty: if not expected_containers_state_changed.empty:
failures.append('expected containers changed state ! : ' failures.append('expected containers changed state ! : '