Test neutron log files rotation

There supposed to be a separate container that rotate all the logs

Change-Id: I29d0304f0d57fd2bc49e35b9d1ace39a036db62a

tobiko/tests/faults/containers/container_ops.py

@@ -19,7 +19,7 @@ import re
 
 from oslo_log import log
 
-from tobiko import fail
+import tobiko
 from tobiko.openstack import topology
 from tobiko.shell import sh
 
@@ -195,17 +195,32 @@ def log_random_msg(node, container, logfile):
     random_msg = ''.join(random.choice(symbols) for i in range(30))
     LOG.debug(f'Trying to print {random_msg} string to {logfile} log file '
               f'in {container} container on {node.name} node')
-    cmd = f"sh -c 'echo {random_msg} >> {logfile}'"
+    log_msg(node, container, logfile, random_msg)
+    return random_msg
+
+
+def log_msg(node, container, logfile, msg):
+    """Print random message to the container log file
+
+    :param node: Node the container is running on
+    :type node: class: tobiko.openstack.topology.OpenStackTopologyNode
+    :param container: Name of the container
+    :type container: string
+    :param logfile: Path to the logfile on the container
+    :type logfile: string
+    :param msg: Message to log
+    :type msg: string
+    """
+    cmd = f"sh -c 'echo {msg} >> {logfile}'"
     error = sh.execute(f'sudo podman exec -it -u root {container} {cmd}',
                        ssh_client=node.ssh_client,
                        expect_exit_status=None).stderr
     if error:
-        fail(f'Cannot edit {logfile} in {container} on {node.name} '
+        tobiko.fail(f'Cannot edit {logfile} in {container} on {node.name} '
-             f'got the following error:\n{error}')
+                    f'got the following error:\n{error}')
-    return random_msg
 
 
-def find_msg_in_file(node, logfile, message):
+def find_msg_in_file(node, logfile, message, rotated=False):
     """Search for the message in the logfile
 
     :param node: Node the container is running on
@@ -214,16 +229,43 @@ def find_msg_in_file(node, logfile, message):
     :type logfile: string
     :param message: Message to search for
     :type message: string
+    :param rotated: Variable to flag that log file has to be rotated
+        so the name will be ended by '.1'
+    :type rotated: bool
     :return: True if message exists in file or False otherwise
     :rtype: bool
     """
-    LOG.debug(f'Searching for {message} in {logfile} on {node.name}')
+    if rotated:
-    result = sh.execute(f'sudo grep -h {message} {logfile}{{,.1}}',
+        suffix = ".1"
+    else:
+        suffix = "{,.1}"
+    LOG.debug(f'Searching for {message} in {logfile}{suffix} on {node.name}')
+    result = sh.execute(f'sudo grep -h {message} {logfile}{suffix}',
                         ssh_client=node.ssh_client,
                         expect_exit_status=None)
     if result.stderr:
-        fail(f'Failed reading {logfile} on {node.name}:\n{result.stderr}')
+        tobiko.fail(f'Failed to read {logfile} on {node.name}:\n'
+                    f'{result.stderr}')
     elif result.stdout.strip() == message:
         return True
     else:
         return False
+
+
+def rotate_logs(node):
+    """Rotate all the container logs using 'logrotate'
+
+    :param node: Node to rotate logs on
+    :type node: class: tobiko.openstack.topology.OpenStackTopologyNode
+    """
+    containers = get_filtered_node_containers(node, ['logrotate.*', ])
+    if not containers:
+        tobiko.skip('No logrotate container has been found')
+    else:
+        container = containers[0]
+    rotate = sh.execute(f'sudo podman exec -it -u root {container} logrotate '
+                        '-f /etc/logrotate-crond.conf',
+                        ssh_client=node.ssh_client,
+                        expect_exit_status=None)
+    if rotate.stderr:
+        tobiko.fail(f'Failed rotate logs on {node.name} node')

tobiko/tests/faults/containers/test_container_log.py

@@ -31,13 +31,45 @@ class LogFilesTest(testtools.TestCase):
         for node in neutron_nodes:
             containers = container_ops.get_node_neutron_containers(node)
             for container in containers:
-                l_file = container_ops.get_container_logfile(node, container)
+                logfile = container_ops.get_container_logfile(node, container)
-                if not l_file:
+                if not logfile:
                     LOG.warning(f'No logfile has been found in {container} '
                                 f'container of {node.name} node')
                     continue
-                log_msg = container_ops.log_random_msg(node, container, l_file)
+                log_msg = container_ops.log_random_msg(node,
-                l_node = f'/var/log/containers/neutron/{l_file.split("/")[-1]}'
+                                                       container,
+                                                       logfile)
+                node_logfile = '/var/log/containers/neutron/'\
+                               f'{logfile.split("/")[-1]}'
                 self.assertTrue(container_ops.find_msg_in_file(node,
-                                                               l_node,
+                                                               node_logfile,
                                                                log_msg))
+
+    def test_neutron_logs_rotate(self):
+        groups = ['controller', 'compute', 'networker']
+        neutron_nodes = container_ops.get_nodes_for_groups(groups)
+        msg = ''
+        for node in neutron_nodes:
+            logfiles = []
+            containers = container_ops.get_node_neutron_containers(node)
+            for container in containers:
+                logfile = container_ops.get_container_logfile(node, container)
+                if not logfile:
+                    LOG.warning(f'No logfile has been found in {container} '
+                                f'container of {node.name} node')
+                    continue
+                logfiles.append(logfile)
+                if not msg:
+                    msg = container_ops.log_random_msg(node,
+                                                       container,
+                                                       logfile)
+                else:
+                    container_ops.log_msg(node, container, logfile, msg)
+            container_ops.rotate_logs(node)
+            for logfile in logfiles:
+                node_logfile = '/var/log/containers/neutron/'\
+                               f'{logfile.split("/")[-1]}'
+                self.assertTrue(container_ops.find_msg_in_file(node,
+                                                               node_logfile,
+                                                               msg,
+                                                               rotated=True))