Shelve name of tests using a heat stack

Tobiko workers can share heat stacks. A test checks whether the stack it
requires already exists and if not, it creates it.
Sometimes, when a test fails, it deletes its stack, affecting other
tests using the same stack, running in parallel on different workers.

This patch saves persistently the name of the tests using a heat stack
using sets.
The test names are added to those sets when a new test is going to use
the stack and are removed from the sets when the test stops using the
stack (or when the test ends).
The cleanup method only deletes the stack if the numnber of tests using
it is zero.

The shelves are cleaned up everytime tobiko is executed with pytest (or
tox)

Change-Id: I92655be072efe8ecb7993c7dbf76ba930b6d9a89

tobiko/__init__.py

@@ -30,6 +30,7 @@ from tobiko.common import _operation
 from tobiko.common import _os
 from tobiko.common import _retry
 from tobiko.common import _select
+from tobiko.common import _shelves
 from tobiko.common import _skip
 from tobiko.common import _time
 from tobiko.common import _utils
@@ -134,6 +135,12 @@ select_uniques = _select.select_uniques
 ObjectNotFound = _select.ObjectNotFound
 MultipleObjectsFound = _select.MultipleObjectsFound
 
+addme_to_shared_resource = _shelves.addme_to_shared_resource
+removeme_from_shared_resource = _shelves.removeme_from_shared_resource
+remove_test_from_all_shared_resources = (
+    _shelves.remove_test_from_all_shared_resources)
+initialize_shelves = _shelves.initialize_shelves
+
 SkipException = _skip.SkipException
 skip_if = _skip.skip_if
 skip_on_error = _skip.skip_on_error

tobiko/common/_case.py

@@ -150,6 +150,7 @@ def enter_test_case(case: TestCase,
             yield
     finally:
         assert case is manager.pop_test_case()
+        tobiko.remove_test_from_all_shared_resources(case.id())
 
 
 def test_case(case: TestCase = None,

tobiko/common/_shelves.py

@@ -0,0 +1,152 @@
+# Copyright 2022 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import dbm
+import os
+import shelve
+
+from oslo_log import log
+
+import tobiko
+
+
+LOG = log.getLogger(__name__)
+TEST_RUN_SHELF = 'test_run'
+
+
+def get_shelves_dir():
+    # ensure the directory exists
+    from tobiko import config
+    shelves_dir = os.path.expanduser(config.CONF.tobiko.common.shelves_dir)
+    return shelves_dir
+
+
+def get_shelf_path(shelf):
+    return os.path.join(get_shelves_dir(), shelf)
+
+
+def addme_to_shared_resource(shelf, resource):
+    shelf_path = get_shelf_path(shelf)
+    # this is needed for unit tests
+    resource = str(resource)
+    testcase_id = tobiko.get_test_case().id()
+    for attempt in tobiko.retry(timeout=10.0,
+                                interval=0.5):
+        try:
+            with shelve.open(shelf_path) as db:
+                if db.get(resource) is None:
+                    db[resource] = set()
+                # the add and remove methods do not work directly on the db
+                auxset = db[resource]
+                auxset.add(testcase_id)
+                db[resource] = auxset
+                return db[resource]
+        except dbm.error:
+            LOG.exception(f"Error accessing shelf {shelf}")
+            if attempt.is_last:
+                raise
+
+
+def removeme_from_shared_resource(shelf, resource):
+    shelf_path = get_shelf_path(shelf)
+    # this is needed for unit tests
+    resource = str(resource)
+    testcase_id = tobiko.get_test_case().id()
+    for attempt in tobiko.retry(timeout=10.0,
+                                interval=0.5):
+        try:
+            with shelve.open(shelf_path) as db:
+                # the add and remove methods do not work directly on the db
+                db[resource] = db.get(resource) or set()
+                if testcase_id in db[resource]:
+                    auxset = db[resource]
+                    auxset.remove(testcase_id)
+                    db[resource] = auxset
+                return db[resource]
+        except dbm.error:
+            LOG.exception(f"Error accessing shelf {shelf}")
+            if attempt.is_last:
+                raise
+
+
+def remove_test_from_shelf_resources(testcase_id, shelf):
+    shelf_path = get_shelf_path(shelf)
+    for attempt in tobiko.retry(timeout=10.0,
+                                interval=0.5):
+        try:
+            with shelve.open(shelf_path) as db:
+                if not db:
+                    return
+                for resource in db.keys():
+                    if testcase_id in db[resource]:
+                        auxset = db[resource]
+                        auxset.remove(testcase_id)
+                        db[resource] = auxset
+                return db
+        except dbm.error as err:
+            LOG.exception(f"Error accessing shelf {shelf}")
+            if "db type could not be determined" in str(err):
+                # remove the filename extension, which depends on the specific
+                # DBM implementation
+                shelf_path = '.'.join(shelf_path.split('.')[:-1])
+            if attempt.is_last:
+                raise
+
+
+def remove_test_from_all_shared_resources(testcase_id):
+    LOG.debug(f'Removing test {testcase_id} from all shelf resources')
+    shelves_dir = get_shelves_dir()
+    for filename in os.listdir(shelves_dir):
+        if TEST_RUN_SHELF not in filename:
+            remove_test_from_shelf_resources(testcase_id, filename)
+
+
+def initialize_shelves():
+    shelves_dir = get_shelves_dir()
+    shelf_path = os.path.join(shelves_dir, TEST_RUN_SHELF)
+    id_key = 'PYTEST_XDIST_TESTRUNUID'
+    test_run_uid = os.environ.get(id_key)
+
+    tobiko.makedirs(shelves_dir)
+
+    # if no PYTEST_XDIST_TESTRUNUID ->
+    #     pytest was executed with only one worker
+    # if tobiko.initialize_shelves() == True ->
+    #    this is the first pytest worker running cleanup_shelves
+    # then, cleanup the shelves directory
+    # else, another worker did it before
+    for attempt in tobiko.retry(timeout=15.0,
+                                interval=0.5):
+        try:
+            with shelve.open(shelf_path) as db:
+                if test_run_uid is None:
+                    LOG.debug("Only one pytest worker - Initializing shelves")
+                elif test_run_uid == db.get(id_key):
+                    LOG.debug("Another pytest worker already initialized "
+                              "the shelves")
+                    return
+                else:
+                    LOG.debug("Initializing shelves for the "
+                              "test run uid %s", test_run_uid)
+                    db[id_key] = test_run_uid
+                for filename in os.listdir(shelves_dir):
+                    if TEST_RUN_SHELF not in filename:
+                        file_path = os.path.join(shelves_dir, filename)
+                        os.unlink(file_path)
+                return
+        except dbm.error:
+            LOG.exception(f"Error accessing shelf {TEST_RUN_SHELF}")
+            if attempt.is_last:
+                raise

tobiko/config.py

@@ -69,7 +69,6 @@ HTTP_OPTIONS = [
     cfg.StrOpt('no_proxy',
                help="Don't use proxy server to connect to listed hosts")]
 
-
 TESTCASE_CONF_GROUP_NAME = "testcase"
 
 TESTCASE_OPTIONS = [
@@ -82,6 +81,14 @@ TESTCASE_OPTIONS = [
                  help=("Timeout (in seconds) used for interrupting test "
                        "runner execution"))]
 
+COMMON_GROUP_NAME = 'common'
+
+COMMON_OPTIONS = [
+    cfg.StrOpt('shelves_dir',
+               default='~/.tobiko/cache/shelves',
+               help=("Default directory where to look for shelves.")),
+]
+
 
 def workspace_config_files(project=None, prog=None):
     project = project or 'tobiko'
@@ -217,6 +224,9 @@ def register_tobiko_options(conf):
     conf.register_opts(
         group=cfg.OptGroup(TESTCASE_CONF_GROUP_NAME), opts=TESTCASE_OPTIONS)
 
+    conf.register_opts(
+        group=cfg.OptGroup(COMMON_GROUP_NAME), opts=COMMON_OPTIONS)
+
     for module_name in CONFIG_MODULES:
         module = importlib.import_module(module_name)
         if hasattr(module, 'register_tobiko_options'):
@@ -235,9 +245,16 @@ def list_testcase_options():
     ]
 
 
+def list_common_options():
+    return [
+        (COMMON_GROUP_NAME, itertools.chain(COMMON_OPTIONS))
+    ]
+
+
 def list_tobiko_options():
     all_options = (list_http_options() +
-                   list_testcase_options())
+                   list_testcase_options() +
+                   list_common_options())
 
     for module_name in CONFIG_MODULES:
         module = importlib.import_module(module_name)

tobiko/openstack/heat/_stack.py

@@ -188,7 +188,9 @@ class HeatStackFixture(tobiko.SharedFixture):
             self.user = keystone.get_user_id(session=self.session)
 
     def setup_stack(self) -> stacks.Stack:
-        return self.create_stack()
+        stack = self.create_stack()
+        tobiko.addme_to_shared_resource(__name__, stack.stack_name)
+        return stack
 
     def get_stack_parameters(self):
         return tobiko.reset_fixture(self.parameters).values
@@ -210,6 +212,8 @@ class HeatStackFixture(tobiko.SharedFixture):
                     if attempt.is_last:
                         raise
 
+                    # the stack shelf counter does not need to be decreased
+                    # here, because it was not increased yet
                     self.delete_stack()
 
                     # It uses a random time sleep to make conflicting
@@ -252,6 +256,8 @@ class HeatStackFixture(tobiko.SharedFixture):
                 LOG.error(f"Stack '{self.stack_name}' (id='{stack.id}') "
                           f"found in '{stack_status}' status (reason="
                           f"'{stack.stack_status_reason}'). Deleting it...")
+                # the stack shelf counter does not need to be decreased here,
+                # because it was not increased yet
                 self.delete_stack(stack_id=stack.id)
 
             self.wait_until_stack_deleted()
@@ -286,12 +292,16 @@ class HeatStackFixture(tobiko.SharedFixture):
         except InvalidStackError as ex:
             LOG.debug(f'Deleting invalid stack (name={self.stack_name}, "'
                       f'"id={stack_id}): {ex}')
+            # the stack shelf counter does not need to be decreased here,
+            # because it was not increased yet
             self.delete_stack(stack_id=stack_id)
             raise
 
         if stack_id != stack.id:
             LOG.debug(f'Deleting duplicate stack (name={self.stack_name}, "'
                       f'"id={stack_id})')
+            # the stack shelf counter does not need to be decreased here,
+            # because it was not increased yet
             self.delete_stack(stack_id=stack_id)
             del stack_id
 
@@ -312,8 +322,14 @@ class HeatStackFixture(tobiko.SharedFixture):
         return resources
 
     def cleanup_fixture(self):
-        self.setup_client()
-        self.cleanup_stack()
+        n_tests_using_stack = len(tobiko.removeme_from_shared_resource(
+            __name__, self.stack_name))
+        if n_tests_using_stack == 0:
+            self.setup_client()
+            self.cleanup_stack()
+        else:
+            LOG.info('Stack %r not deleted because %d tests are using it',
+                     self.stack_name, n_tests_using_stack)
 
     def cleanup_stack(self):
         self.delete_stack()

tobiko/run/_result.py

@@ -59,6 +59,7 @@ class TestResult(unittest.TextTestResult):
         super().stopTestRun()
         actual_test = tobiko.pop_test_case()
         assert actual_test == test
+        tobiko.remove_test_from_all_shared_resources(test.id())
 
 
 class TextIOWrapper(io.TextIOWrapper):

tobiko/tests/conftest.py

@@ -231,3 +231,8 @@ def pytest_runtest_call(item):
     # pylint: disable=unused-argument
     check_test_runner_timeout()
     yield
+
+
+@pytest.fixture(scope="session", autouse=True)
+def cleanup_shelves():
+    tobiko.initialize_shelves()

tobiko/tests/unit/openstack/heat/test_stack.py

@@ -261,9 +261,12 @@ class HeatStackFixtureTest(openstack.OpenstackTest):
     def test_cleanup(self):
         client = MockClient()
         client.stacks.get.return_value = None
-        stack = MyStack(client=client)
+        stack = MyStackWithStackName(client=client)
+        stack_name = stack.stack_name
+        tobiko.addme_to_shared_resource(
+            'tobiko.openstack.heat._stack', stack_name)
         stack.cleanUp()
-        client.stacks.delete.assert_called_once_with(stack.stack_name)
+        client.stacks.delete.assert_called_once_with(stack_name)
 
     def test_outputs(self):
         stack = mock_stack(status='CREATE_COMPLETE',