Introduce new BaseResourceFixture class

This new class is base for both: HeatStackFixture and ResourceFixture
classes. It contains some common pieces of code like e.g. ensuring
that quota is set as required by the fixture.

This is done because fixture StatelessSecurityGroup, which don't use
HeatStackFixture needs to set quota for security groups in Neutron but
ensuring neutron (and nova) quotas are set correctly were done only in
the HeatStackFixture class so far.

Change-Id: Id3d3207f098853469bef87020fc017bec5aaba93

tobiko/openstack/base/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2024 Red Hat, Inc.
+#
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

tobiko/openstack/base/_fixture.py

@@ -14,18 +14,94 @@
 from __future__ import absolute_import
 
 import abc
+import collections
 import typing
 
 from oslo_log import log
 
 import tobiko
 from tobiko import config
+from tobiko.openstack import keystone
+from tobiko.openstack.neutron import _quota_set as neutron_quota
+from tobiko.openstack.nova import _quota_set as nova_quota
 
 
 LOG = log.getLogger(__name__)
 
 
-class ResourceFixture(tobiko.SharedFixture, abc.ABC):
+class InvalidFixtureError(tobiko.TobikoException):
+    message = "invalid fixture {name!r}"
+
+
+class BaseResourceFixture(tobiko.SharedFixture):
+    """Base class for fixtures both types: those which uses heat stacks and
+    those which are not.
+    """
+    client: keystone.KeystoneClient = None
+    project: typing.Optional[str] = None
+    user: typing.Optional[str] = None
+
+    def setup_fixture(self):
+        self.setup_client()
+        self.setup_project()
+        self.setup_user()
+
+    def setup_project(self):
+        if self.project is None:
+            self.project = keystone.get_project_id(session=self.session)
+
+    def setup_user(self):
+        if self.user is None:
+            self.user = keystone.get_user_id(session=self.session)
+
+    @property
+    def session(self):
+        return self.setup_client().session
+
+    def setup_client(self) -> keystone.KeystoneClient:
+        client = self.client
+        # NOTE(slaweq): it seems that due to bug
+        # https://github.com/python/mypy/issues/11673
+        # in mypy this line is causing arg-type error so lets
+        # ignore it for now
+        if not isinstance(
+                client, keystone.KeystoneClient):  # type: ignore[arg-type]
+            self.client = client = keystone.keystone_client(self.client)
+        return client
+
+    def ensure_quota_limits(self):
+        """Ensures quota limits before creating a new stack
+        """
+        try:
+            self.ensure_neutron_quota_limits()
+            self.ensure_nova_quota_limits()
+        except (nova_quota.EnsureNovaQuotaLimitsError,
+                neutron_quota.EnsureNeutronQuotaLimitsError) as ex:
+            raise InvalidFixtureError(name=self.fixture_name) from ex
+
+    def ensure_neutron_quota_limits(self):
+        required_quota_set = self.neutron_required_quota_set
+        if required_quota_set:
+            neutron_quota.ensure_neutron_quota_limits(project=self.project,
+                                                      **required_quota_set)
+
+    def ensure_nova_quota_limits(self):
+        required_quota_set = self.nova_required_quota_set
+        if required_quota_set:
+            nova_quota.ensure_nova_quota_limits(project=self.project,
+                                                user=self.user,
+                                                **required_quota_set)
+
+    @property
+    def neutron_required_quota_set(self) -> typing.Dict[str, int]:
+        return collections.defaultdict(int)
+
+    @property
+    def nova_required_quota_set(self) -> typing.Dict[str, int]:
+        return collections.defaultdict(int)
+
+
+class ResourceFixture(BaseResourceFixture, abc.ABC):
     """Base class for fixtures intended to manage Openstack resources not
     created using Heat, but with openstacksdk or other component clients (such
     as neutronclient, novaclient, manilaclient, etc).
@@ -41,11 +117,11 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC):
     Child classes must define any other attributes required by the
     resource_create, resource_delete and resource_find methods. Examples:
     prefixes and default_prefixlen are needed for subnet_pools; description and
-    rules are needed for secutiry_groups; etc.
+    rules are needed for security_groups; etc.
 
     Child classes must define the resource_create, resource_delete and
     resource_find methods. In case of resource_create and resource_find, they
-    should return an object with the type defined for self._resouce.
+    should return an object with the type defined for self._resource.
 
     Child classes may optionally implement simple properties to access to
     resource_id and resource using a more representative name (these properties
@@ -84,6 +160,7 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC):
         pass
 
     def setup_fixture(self):
+        super().setup_fixture()
         self.name = self.fixture_name
         if config.get_bool_env('TOBIKO_PREVENT_CREATE'):
             LOG.debug("%r should have been already created: %r",
@@ -98,6 +175,9 @@ class ResourceFixture(tobiko.SharedFixture, abc.ABC):
             tobiko.addme_to_shared_resource(__name__, self.name)
 
     def try_create_resource(self):
+        # Ensure quota limits are OK just in time before start creating
+        # a new stack
+        self.ensure_quota_limits()
         if not self.resource:
             self._resource = self.resource_create()
 

tobiko/openstack/heat/__init__.py

@@ -41,7 +41,6 @@ HeatStackNotFound = _stack.HeatStackNotFound
 heat_stack_parameters = _stack.heat_stack_parameters
 find_stack = _stack.find_stack
 list_stacks = _stack.list_stacks
-InvalidStackError = _stack.InvalidStackError
 INIT_IN_PROGRESS = _stack.INIT_IN_PROGRESS
 INIT_COMPLETE = _stack.INIT_COMPLETE
 CREATE_IN_PROGRESS = _stack.CREATE_IN_PROGRESS

tobiko/openstack/heat/_stack.py

@@ -13,7 +13,6 @@
 #    under the License.
 from __future__ import absolute_import
 
-import collections
 from collections import abc
 import random
 import time
@@ -28,8 +27,7 @@ from tobiko import config
 from tobiko.openstack.heat import _client
 from tobiko.openstack.heat import _template
 from tobiko.openstack import keystone
-from tobiko.openstack import neutron
+from tobiko.openstack.base import _fixture as base_fixture
-from tobiko.openstack import nova
 
 
 LOG = log.getLogger(__name__)
@@ -110,7 +108,7 @@ def find_stack(client: _client.HeatClientType = None,
 
 
 @keystone.skip_unless_has_keystone_credentials()
-class HeatStackFixture(tobiko.SharedFixture):
+class HeatStackFixture(base_fixture.BaseResourceFixture):
     """Manages Heat stacks."""
 
     client: _client.HeatClientType = None
@@ -124,8 +122,6 @@ class HeatStackFixture(tobiko.SharedFixture):
     stack: typing.Optional[StackType] = None
     stack_name: typing.Optional[str] = None
     parameters: typing.Optional['HeatStackParametersFixture'] = None
-    project: typing.Optional[str] = None
-    user: typing.Optional[str] = None
     output_needs_stack_complete: bool = True
 
     def __init__(
@@ -150,12 +146,10 @@ class HeatStackFixture(tobiko.SharedFixture):
             self.wait_interval = wait_interval
 
     def setup_fixture(self):
+        super().setup_fixture()
         self.setup_stack_name()
         self.setup_template()
         self.setup_parameters()
-        self.setup_client()
-        self.setup_project()
-        self.setup_user()
         self.setup_stack()
 
     def setup_template(self):
@@ -180,14 +174,6 @@ class HeatStackFixture(tobiko.SharedFixture):
     def session(self):
         return self.setup_client().http_client.session
 
-    def setup_project(self):
-        if self.project is None:
-            self.project = keystone.get_project_id(session=self.session)
-
-    def setup_user(self):
-        if self.user is None:
-            self.user = keystone.get_user_id(session=self.session)
-
     def setup_stack(self) -> stacks.Stack:
         stack = self.create_stack()
         tobiko.addme_to_shared_resource(__name__, stack.stack_name)
@@ -207,7 +193,7 @@ class HeatStackFixture(tobiko.SharedFixture):
                 try:
                     stack = self.try_create_stack()
                     break
-                except InvalidStackError:
+                except base_fixture.InvalidFixtureError:
                     LOG.exception(f"Error creating stack '{self.stack_name}'",
                                   exc_info=1)
                     if attempt.is_last:
@@ -290,7 +276,7 @@ class HeatStackFixture(tobiko.SharedFixture):
                   f"id='{stack_id}'.")
         try:
             stack = self.validate_created_stack()
-        except InvalidStackError as ex:
+        except base_fixture.InvalidFixtureError as ex:
             LOG.debug(f'Deleting invalid stack (name={self.stack_name}, "'
                       f'"id={stack_id}): {ex}')
             # the stack shelf counter does not need to be decreased here,
@@ -517,37 +503,6 @@ class HeatStackFixture(tobiko.SharedFixture):
         message = "Object {!r} has no attribute {!r}".format(self, name)
         raise AttributeError(message)
 
-    def ensure_quota_limits(self):
-        """Ensures quota limits before creating a new stack
-        """
-        try:
-            self.ensure_neutron_quota_limits()
-            self.ensure_nova_quota_limits()
-        except (nova.EnsureNovaQuotaLimitsError,
-                neutron.EnsureNeutronQuotaLimitsError) as ex:
-            raise InvalidStackError(name=self.stack_name) from ex
-
-    def ensure_neutron_quota_limits(self):
-        required_quota_set = self.neutron_required_quota_set
-        if required_quota_set:
-            neutron.ensure_neutron_quota_limits(project=self.project,
-                                                **required_quota_set)
-
-    def ensure_nova_quota_limits(self):
-        required_quota_set = self.nova_required_quota_set
-        if required_quota_set:
-            nova.ensure_nova_quota_limits(project=self.project,
-                                          user=self.user,
-                                          **required_quota_set)
-
-    @property
-    def neutron_required_quota_set(self) -> typing.Dict[str, int]:
-        return collections.defaultdict(int)
-
-    @property
-    def nova_required_quota_set(self) -> typing.Dict[str, int]:
-        return collections.defaultdict(int)
-
 
 class HeatStackKeyError(tobiko.TobikoException):
     message = "key {key!r} not found in stack {name!r}"
@@ -717,11 +672,7 @@ class HeatStackNotFound(HeatStackError):
     message = "stack {name!r} not found"
 
 
-class InvalidStackError(HeatStackError):
+class InvalidHeatStackStatus(base_fixture.InvalidFixtureError):
-    message = "invalid stack {name!r}"
-
-
-class InvalidHeatStackStatus(InvalidStackError):
     message = ("stack {name!r} status {observed!r} not in {expected!r}\n"
                "{status_reason!s}")
 

tobiko/openstack/stacks/_neutron.py

@@ -26,8 +26,8 @@ import tobiko
 from tobiko import config
 from tobiko.openstack import heat
 from tobiko.openstack import neutron
+from tobiko.openstack.base import _fixture as base_fixture
 from tobiko.openstack.stacks import _hot
-from tobiko.openstack.stacks import _fixture
 from tobiko.shell import ip
 from tobiko.shell import sh
 from tobiko.shell import ssh
@@ -284,7 +284,7 @@ class RouterNoSnatStackFixture(RouterStackFixture):
 
 
 @neutron.skip_if_missing_networking_extensions('subnet_allocation')
-class SubnetPoolFixture(_fixture.ResourceFixture):
+class SubnetPoolFixture(base_fixture.ResourceFixture):
     """Neutron Subnet Pool Fixture.
 
     A subnet pool is a dependency of network fixtures with either IPv4 or
@@ -583,7 +583,7 @@ class SecurityGroupsFixture(heat.HeatStackFixture):
 
 
 @neutron.skip_if_missing_networking_extensions('stateful-security-group')
-class StatelessSecurityGroupFixture(_fixture.ResourceFixture):
+class StatelessSecurityGroupFixture(base_fixture.ResourceFixture):
     """Neutron Stateless Security Group Fixture.
 
     This SG will by default allow SSH and ICMP to the instance and also
@@ -615,6 +615,12 @@ class StatelessSecurityGroupFixture(_fixture.ResourceFixture):
         self.description = description or self.description
         self.rules = rules or self.rules
 
+    @property
+    def neutron_required_quota_set(self) -> typing.Dict[str, int]:
+        requirements = super().neutron_required_quota_set
+        requirements['security_group'] += 1
+        return requirements
+
     @property
     def security_group_id(self):
         return self.resource_id

tobiko/openstack/stacks/_nova.py

@@ -28,6 +28,7 @@ from tobiko.openstack import glance
 from tobiko.openstack import heat
 from tobiko.openstack import neutron
 from tobiko.openstack import nova
+from tobiko.openstack.base import _fixture as base_fixture
 from tobiko.openstack.stacks import _hot
 from tobiko.openstack.stacks import _neutron
 from tobiko.shell import curl
@@ -271,7 +272,8 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
                 self.validate_different_host_scheduler_hints(
                     hypervisor=hypervisor)
             except nova.MigrateServerError as ex:
-                raise heat.InvalidStackError(name=self.stack_name) from ex
+                raise base_fixture.InvalidFixtureError(
+                    name=self.stack_name) from ex
 
     def validate_same_host_scheduler_hints(self, hypervisor):
         if self.same_host:

tobiko/tests/scenario/nova/test_server.py

@@ -22,10 +22,10 @@ import pytest
 import testtools
 
 import tobiko
-from tobiko.openstack import heat
 from tobiko.openstack import keystone
 from tobiko.openstack import nova
 from tobiko.openstack import stacks
+from tobiko.openstack.base import _fixture as base_fixture
 from tobiko.shell import ping
 
 
@@ -177,7 +177,7 @@ class CirrosServerStackFixture(stacks.CirrosServerStackFixture):
                 try:
                     nova.activate_server(server)
                 except nova.WaitForServerStatusTimeout as ex:
-                    raise heat.InvalidStackError(
+                    raise base_fixture.InvalidFixtureError(
                         name=self.stack_name) from ex
         return stack