From 41a47bf8cf31ff132c80f045aca104037169764f Mon Sep 17 00:00:00 2001 From: Eduardo Olivares Date: Tue, 29 Nov 2022 10:36:32 +0100 Subject: [PATCH] Shelve name of tests using a heat stack Tobiko workers can share heat stacks. A test checks whether the stack it requires already exists and if not, it creates it. Sometimes, when a test fails, it deletes its stack, affecting other tests using the same stack, running in parallel on different workers. This patch saves persistently the name of the tests using a heat stack using sets. The test names are added to those sets when a new test is going to use the stack and are removed from the sets when the test stops using the stack (or when the test ends). The cleanup method only deletes the stack if the numnber of tests using it is zero. The shelves are cleaned up everytime tobiko is executed with pytest (or tox) Change-Id: I92655be072efe8ecb7993c7dbf76ba930b6d9a89 --- tobiko/__init__.py | 7 + tobiko/common/_case.py | 1 + tobiko/common/_shelves.py | 152 ++++++++++++++++++ tobiko/config.py | 21 ++- tobiko/openstack/heat/_stack.py | 22 ++- tobiko/run/_result.py | 1 + tobiko/tests/conftest.py | 5 + .../tests/unit/openstack/heat/test_stack.py | 7 +- 8 files changed, 209 insertions(+), 7 deletions(-) create mode 100644 tobiko/common/_shelves.py diff --git a/tobiko/__init__.py b/tobiko/__init__.py index 19a174bc0..58dfb1178 100644 --- a/tobiko/__init__.py +++ b/tobiko/__init__.py @@ -30,6 +30,7 @@ from tobiko.common import _operation from tobiko.common import _os from tobiko.common import _retry from tobiko.common import _select +from tobiko.common import _shelves from tobiko.common import _skip from tobiko.common import _time from tobiko.common import _utils @@ -134,6 +135,12 @@ select_uniques = _select.select_uniques ObjectNotFound = _select.ObjectNotFound MultipleObjectsFound = _select.MultipleObjectsFound +addme_to_shared_resource = _shelves.addme_to_shared_resource +removeme_from_shared_resource = _shelves.removeme_from_shared_resource +remove_test_from_all_shared_resources = ( + _shelves.remove_test_from_all_shared_resources) +initialize_shelves = _shelves.initialize_shelves + SkipException = _skip.SkipException skip_if = _skip.skip_if skip_on_error = _skip.skip_on_error diff --git a/tobiko/common/_case.py b/tobiko/common/_case.py index 61ac9ac0e..20e44a293 100644 --- a/tobiko/common/_case.py +++ b/tobiko/common/_case.py @@ -150,6 +150,7 @@ def enter_test_case(case: TestCase, yield finally: assert case is manager.pop_test_case() + tobiko.remove_test_from_all_shared_resources(case.id()) def test_case(case: TestCase = None, diff --git a/tobiko/common/_shelves.py b/tobiko/common/_shelves.py new file mode 100644 index 000000000..113b98a0e --- /dev/null +++ b/tobiko/common/_shelves.py @@ -0,0 +1,152 @@ +# Copyright 2022 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import dbm +import os +import shelve + +from oslo_log import log + +import tobiko + + +LOG = log.getLogger(__name__) +TEST_RUN_SHELF = 'test_run' + + +def get_shelves_dir(): + # ensure the directory exists + from tobiko import config + shelves_dir = os.path.expanduser(config.CONF.tobiko.common.shelves_dir) + return shelves_dir + + +def get_shelf_path(shelf): + return os.path.join(get_shelves_dir(), shelf) + + +def addme_to_shared_resource(shelf, resource): + shelf_path = get_shelf_path(shelf) + # this is needed for unit tests + resource = str(resource) + testcase_id = tobiko.get_test_case().id() + for attempt in tobiko.retry(timeout=10.0, + interval=0.5): + try: + with shelve.open(shelf_path) as db: + if db.get(resource) is None: + db[resource] = set() + # the add and remove methods do not work directly on the db + auxset = db[resource] + auxset.add(testcase_id) + db[resource] = auxset + return db[resource] + except dbm.error: + LOG.exception(f"Error accessing shelf {shelf}") + if attempt.is_last: + raise + + +def removeme_from_shared_resource(shelf, resource): + shelf_path = get_shelf_path(shelf) + # this is needed for unit tests + resource = str(resource) + testcase_id = tobiko.get_test_case().id() + for attempt in tobiko.retry(timeout=10.0, + interval=0.5): + try: + with shelve.open(shelf_path) as db: + # the add and remove methods do not work directly on the db + db[resource] = db.get(resource) or set() + if testcase_id in db[resource]: + auxset = db[resource] + auxset.remove(testcase_id) + db[resource] = auxset + return db[resource] + except dbm.error: + LOG.exception(f"Error accessing shelf {shelf}") + if attempt.is_last: + raise + + +def remove_test_from_shelf_resources(testcase_id, shelf): + shelf_path = get_shelf_path(shelf) + for attempt in tobiko.retry(timeout=10.0, + interval=0.5): + try: + with shelve.open(shelf_path) as db: + if not db: + return + for resource in db.keys(): + if testcase_id in db[resource]: + auxset = db[resource] + auxset.remove(testcase_id) + db[resource] = auxset + return db + except dbm.error as err: + LOG.exception(f"Error accessing shelf {shelf}") + if "db type could not be determined" in str(err): + # remove the filename extension, which depends on the specific + # DBM implementation + shelf_path = '.'.join(shelf_path.split('.')[:-1]) + if attempt.is_last: + raise + + +def remove_test_from_all_shared_resources(testcase_id): + LOG.debug(f'Removing test {testcase_id} from all shelf resources') + shelves_dir = get_shelves_dir() + for filename in os.listdir(shelves_dir): + if TEST_RUN_SHELF not in filename: + remove_test_from_shelf_resources(testcase_id, filename) + + +def initialize_shelves(): + shelves_dir = get_shelves_dir() + shelf_path = os.path.join(shelves_dir, TEST_RUN_SHELF) + id_key = 'PYTEST_XDIST_TESTRUNUID' + test_run_uid = os.environ.get(id_key) + + tobiko.makedirs(shelves_dir) + + # if no PYTEST_XDIST_TESTRUNUID -> + # pytest was executed with only one worker + # if tobiko.initialize_shelves() == True -> + # this is the first pytest worker running cleanup_shelves + # then, cleanup the shelves directory + # else, another worker did it before + for attempt in tobiko.retry(timeout=15.0, + interval=0.5): + try: + with shelve.open(shelf_path) as db: + if test_run_uid is None: + LOG.debug("Only one pytest worker - Initializing shelves") + elif test_run_uid == db.get(id_key): + LOG.debug("Another pytest worker already initialized " + "the shelves") + return + else: + LOG.debug("Initializing shelves for the " + "test run uid %s", test_run_uid) + db[id_key] = test_run_uid + for filename in os.listdir(shelves_dir): + if TEST_RUN_SHELF not in filename: + file_path = os.path.join(shelves_dir, filename) + os.unlink(file_path) + return + except dbm.error: + LOG.exception(f"Error accessing shelf {TEST_RUN_SHELF}") + if attempt.is_last: + raise diff --git a/tobiko/config.py b/tobiko/config.py index 5fcb17c80..42d9630f4 100644 --- a/tobiko/config.py +++ b/tobiko/config.py @@ -69,7 +69,6 @@ HTTP_OPTIONS = [ cfg.StrOpt('no_proxy', help="Don't use proxy server to connect to listed hosts")] - TESTCASE_CONF_GROUP_NAME = "testcase" TESTCASE_OPTIONS = [ @@ -82,6 +81,14 @@ TESTCASE_OPTIONS = [ help=("Timeout (in seconds) used for interrupting test " "runner execution"))] +COMMON_GROUP_NAME = 'common' + +COMMON_OPTIONS = [ + cfg.StrOpt('shelves_dir', + default='~/.tobiko/cache/shelves', + help=("Default directory where to look for shelves.")), +] + def workspace_config_files(project=None, prog=None): project = project or 'tobiko' @@ -217,6 +224,9 @@ def register_tobiko_options(conf): conf.register_opts( group=cfg.OptGroup(TESTCASE_CONF_GROUP_NAME), opts=TESTCASE_OPTIONS) + conf.register_opts( + group=cfg.OptGroup(COMMON_GROUP_NAME), opts=COMMON_OPTIONS) + for module_name in CONFIG_MODULES: module = importlib.import_module(module_name) if hasattr(module, 'register_tobiko_options'): @@ -235,9 +245,16 @@ def list_testcase_options(): ] +def list_common_options(): + return [ + (COMMON_GROUP_NAME, itertools.chain(COMMON_OPTIONS)) + ] + + def list_tobiko_options(): all_options = (list_http_options() + - list_testcase_options()) + list_testcase_options() + + list_common_options()) for module_name in CONFIG_MODULES: module = importlib.import_module(module_name) diff --git a/tobiko/openstack/heat/_stack.py b/tobiko/openstack/heat/_stack.py index 72e1fb64a..18c64d70c 100644 --- a/tobiko/openstack/heat/_stack.py +++ b/tobiko/openstack/heat/_stack.py @@ -188,7 +188,9 @@ class HeatStackFixture(tobiko.SharedFixture): self.user = keystone.get_user_id(session=self.session) def setup_stack(self) -> stacks.Stack: - return self.create_stack() + stack = self.create_stack() + tobiko.addme_to_shared_resource(__name__, stack.stack_name) + return stack def get_stack_parameters(self): return tobiko.reset_fixture(self.parameters).values @@ -210,6 +212,8 @@ class HeatStackFixture(tobiko.SharedFixture): if attempt.is_last: raise + # the stack shelf counter does not need to be decreased + # here, because it was not increased yet self.delete_stack() # It uses a random time sleep to make conflicting @@ -252,6 +256,8 @@ class HeatStackFixture(tobiko.SharedFixture): LOG.error(f"Stack '{self.stack_name}' (id='{stack.id}') " f"found in '{stack_status}' status (reason=" f"'{stack.stack_status_reason}'). Deleting it...") + # the stack shelf counter does not need to be decreased here, + # because it was not increased yet self.delete_stack(stack_id=stack.id) self.wait_until_stack_deleted() @@ -286,12 +292,16 @@ class HeatStackFixture(tobiko.SharedFixture): except InvalidStackError as ex: LOG.debug(f'Deleting invalid stack (name={self.stack_name}, "' f'"id={stack_id}): {ex}') + # the stack shelf counter does not need to be decreased here, + # because it was not increased yet self.delete_stack(stack_id=stack_id) raise if stack_id != stack.id: LOG.debug(f'Deleting duplicate stack (name={self.stack_name}, "' f'"id={stack_id})') + # the stack shelf counter does not need to be decreased here, + # because it was not increased yet self.delete_stack(stack_id=stack_id) del stack_id @@ -312,8 +322,14 @@ class HeatStackFixture(tobiko.SharedFixture): return resources def cleanup_fixture(self): - self.setup_client() - self.cleanup_stack() + n_tests_using_stack = len(tobiko.removeme_from_shared_resource( + __name__, self.stack_name)) + if n_tests_using_stack == 0: + self.setup_client() + self.cleanup_stack() + else: + LOG.info('Stack %r not deleted because %d tests are using it', + self.stack_name, n_tests_using_stack) def cleanup_stack(self): self.delete_stack() diff --git a/tobiko/run/_result.py b/tobiko/run/_result.py index 7437063c7..9f4e6f46a 100644 --- a/tobiko/run/_result.py +++ b/tobiko/run/_result.py @@ -59,6 +59,7 @@ class TestResult(unittest.TextTestResult): super().stopTestRun() actual_test = tobiko.pop_test_case() assert actual_test == test + tobiko.remove_test_from_all_shared_resources(test.id()) class TextIOWrapper(io.TextIOWrapper): diff --git a/tobiko/tests/conftest.py b/tobiko/tests/conftest.py index bdf16d214..d13381625 100644 --- a/tobiko/tests/conftest.py +++ b/tobiko/tests/conftest.py @@ -231,3 +231,8 @@ def pytest_runtest_call(item): # pylint: disable=unused-argument check_test_runner_timeout() yield + + +@pytest.fixture(scope="session", autouse=True) +def cleanup_shelves(): + tobiko.initialize_shelves() diff --git a/tobiko/tests/unit/openstack/heat/test_stack.py b/tobiko/tests/unit/openstack/heat/test_stack.py index fdd62c889..ffeb6f50f 100644 --- a/tobiko/tests/unit/openstack/heat/test_stack.py +++ b/tobiko/tests/unit/openstack/heat/test_stack.py @@ -261,9 +261,12 @@ class HeatStackFixtureTest(openstack.OpenstackTest): def test_cleanup(self): client = MockClient() client.stacks.get.return_value = None - stack = MyStack(client=client) + stack = MyStackWithStackName(client=client) + stack_name = stack.stack_name + tobiko.addme_to_shared_resource( + 'tobiko.openstack.heat._stack', stack_name) stack.cleanUp() - client.stacks.delete.assert_called_once_with(stack.stack_name) + client.stacks.delete.assert_called_once_with(stack_name) def test_outputs(self): stack = mock_stack(status='CREATE_COMPLETE',