x/tobiko
Code Issues Proposed changes

Resolve network stack creation concurrency issues

Browse Source 
Tobiko tests are usually executed with several workers running in
parallel and they reuse shared resources such as networks.
Concurrency issues have been reproduced when several workers are
creating a common network stack.
This patch uses lockeutils to prevent several workers to try to create
the same network at the same time.

Change-Id: I26410654c9a9f4846bfcc1314386a035547f06a0
This commit is contained in:
Eduardo Olivares 2023-07-07 17:11:39 +02:00
parent e4ae45eb3c
commit 5d2a5574ea
10 changed files with 65 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tobiko/__init__.py
View File

@ -174,3 +174,4 @@ load_yaml = _yaml.load_yaml
from tobiko import config # noqa
config.init_config()
LOCK_DIR = os.path.expanduser(config.CONF.tobiko.common.lock_dir)

1
tobiko/openstack/stacks/__init__.py
View File

@ -54,6 +54,7 @@ L3haDifferentHostServerStackFixture = _l3ha.L3haDifferentHostServerStackFixture
L3haSameHostServerStackFixture = _l3ha.L3haSameHostServerStackFixture
FloatingIpStackFixture = _neutron.FloatingIpStackFixture
NetworkBaseStackFixture = _neutron.NetworkBaseStackFixture
NetworkStackFixture = _neutron.NetworkStackFixture
NetworkWithNetMtuWriteStackFixture = (
_neutron.NetworkWithNetMtuWriteStackFixture)

4
tobiko/openstack/stacks/_cirros.py
View File

@ -14,7 +14,6 @@
from __future__ import absolute_import
import io
import os
import typing
from oslo_concurrency import lockutils
@ -32,7 +31,6 @@ import tobiko.tripleo
from tobiko.openstack.stacks import _hot
CONF = config.CONF
LOCK_DIR = os.path.expanduser(CONF.tobiko.common.lock_dir)
CIRROS_IMAGE_VERSION = '0.5.2'
@ -58,7 +56,7 @@ class CirrosImageFixture(glance.URLGlanceImageFixture):
'pubkeys': ['rsa-sha2-256', 'rsa-sha2-512']}
@lockutils.synchronized(
'cirros_image_setup_fixture', external=True, lock_path=LOCK_DIR)
'cirros_image_setup_fixture', external=True, lock_path=tobiko.LOCK_DIR)
def setup_fixture(self):
super(CirrosImageFixture, self).setup_fixture()

9
tobiko/openstack/stacks/_l3ha.py
View File

@ -13,6 +13,8 @@
# under the License.
from __future__ import absolute_import
from oslo_concurrency import lockutils
import tobiko
from tobiko.openstack import neutron
from tobiko.openstack.stacks import _cirros
@ -26,9 +28,14 @@ class L3haRouterStackFixture(_neutron.RouterStackFixture):
@neutron.skip_if_missing_networking_extensions('l3-ha')
class L3haNetworkStackFixture(_neutron.NetworkStackFixture):
class L3haNetworkStackFixture(_neutron.NetworkBaseStackFixture):
gateway_stack = tobiko.required_fixture(L3haRouterStackFixture)
@lockutils.synchronized(
'create_l3ha_network_stack', external=True, lock_path=tobiko.LOCK_DIR)
def setup_stack(self):
super().setup_stack()
@neutron.skip_if_missing_networking_extensions('l3-ha')
class L3haServerStackFixture(_cirros.CirrosServerStackFixture):

30
tobiko/openstack/stacks/_neutron.py
View File

@ -16,7 +16,6 @@
from __future__ import absolute_import
import json
import os
import typing
import netaddr
@ -35,7 +34,6 @@ from tobiko.shell import ssh
CONF = config.CONF
LOG = log.getLogger(__name__)
LOCK_DIR = os.path.expanduser(CONF.tobiko.common.lock_dir)
class ExternalNetworkStackFixture(heat.HeatStackFixture):
@ -334,7 +332,7 @@ class SubnetPoolFixture(tobiko.SharedFixture):
tobiko.addme_to_shared_resource(__name__, self.name)
@lockutils.synchronized(
'create_subnet_pool', external=True, lock_path=LOCK_DIR)
'create_subnet_pool', external=True, lock_path=tobiko.LOCK_DIR)
def try_create_subnet_pool(self):
if not self.subnet_pool:
self._subnet_pool = neutron.create_subnet_pool(
@ -382,7 +380,7 @@ class SubnetPoolIPv6Fixture(SubnetPoolFixture):
@neutron.skip_if_missing_networking_extensions('port-security')
class NetworkStackFixture(heat.HeatStackFixture):
class NetworkBaseStackFixture(heat.HeatStackFixture):
"""Heat stack for creating internal network with a router to external"""
subnet_pools_ipv4_stack = (tobiko.required_fixture(SubnetPoolFixture)
if bool(CONF.tobiko.neutron.ipv4_cidr)
@ -576,13 +574,31 @@ class NetworkStackFixture(heat.HeatStackFixture):
predicate=fixture.is_router_distributed)
class NetworkNoFipStackFixture(NetworkStackFixture):
class NetworkStackFixture(NetworkBaseStackFixture):
@lockutils.synchronized(
'create_network_stack', external=True, lock_path=tobiko.LOCK_DIR)
def setup_stack(self):
super().setup_stack()
class NetworkNoFipStackFixture(NetworkBaseStackFixture):
"""Extra Network Stack where VMs will not have FIPs"""
gateway_stack = tobiko.required_fixture(RouterNoSnatStackFixture)
@lockutils.synchronized(
'create_network_nofip_stack', external=True, lock_path=tobiko.LOCK_DIR)
def setup_stack(self):
super().setup_stack()
@neutron.skip_if_missing_networking_extensions('net-mtu-writable')
class NetworkWithNetMtuWriteStackFixture(NetworkStackFixture):
class NetworkWithNetMtuWriteStackFixture(NetworkBaseStackFixture):
@lockutils.synchronized(
'create_network_withnetmtuwrite_stack',
external=True, lock_path=tobiko.LOCK_DIR)
def setup_stack(self):
super().setup_stack()
@property
def custom_mtu_size(self):
@ -661,7 +677,7 @@ class StatelessSecurityGroupFixture(tobiko.SharedFixture):
tobiko.addme_to_shared_resource(__name__, self.name)
@lockutils.synchronized(
'create_security_group', external=True, lock_path=LOCK_DIR)
'create_security_group', external=True, lock_path=tobiko.LOCK_DIR)
def try_create_security_group(self):
if not self.security_group:
self._security_group = neutron.create_security_group(

9
tobiko/openstack/stacks/_qos.py
View File

@ -15,6 +15,8 @@
# under the License.
from __future__ import absolute_import
from oslo_concurrency import lockutils
import tobiko
from tobiko import config
from tobiko.openstack import heat
@ -44,7 +46,7 @@ class QosPolicyStackFixture(heat.HeatStackFixture):
@neutron.skip_if_missing_networking_extensions('qos')
class QosNetworkStackFixture(_neutron.NetworkStackFixture):
class QosNetworkStackFixture(_neutron.NetworkBaseStackFixture):
#: stack with the qos policy for the network
qos_stack = tobiko.required_fixture(QosPolicyStackFixture)
@ -56,6 +58,11 @@ class QosNetworkStackFixture(_neutron.NetworkStackFixture):
value_specs = super().network_value_specs
return dict(value_specs, qos_policy_id=self.qos_stack.qos_policy_id)
@lockutils.synchronized(
'create_qos_network_stack', external=True, lock_path=tobiko.LOCK_DIR)
def setup_stack(self):
super().setup_stack()
class QosServerStackFixture(_ubuntu.UbuntuServerStackFixture):
#: stack with the network with a qos policy

5
tobiko/openstack/stacks/_ubuntu.py
View File

@ -13,7 +13,6 @@
# under the License.
from __future__ import absolute_import
import os
import typing
from oslo_concurrency import lockutils
@ -28,7 +27,6 @@ from tobiko.shell import sh
CONF = config.CONF
LOCK_DIR = os.path.expanduser(CONF.tobiko.common.lock_dir)
class UbuntuMinimalImageFixture(glance.FileGlanceImageFixture):
@ -43,7 +41,8 @@ class UbuntuMinimalImageFixture(glance.FileGlanceImageFixture):
is_reachable_timeout = CONF.tobiko.nova.ubuntu_is_reachable_timeout
@lockutils.synchronized(
'ubuntu_minimal_setup_fixture', external=True, lock_path=LOCK_DIR)
'ubuntu_minimal_setup_fixture',
external=True, lock_path=tobiko.LOCK_DIR)
def setup_fixture(self):
super(UbuntuMinimalImageFixture, self).setup_fixture()

8
tobiko/openstack/stacks/_vlan.py
View File

@ -19,6 +19,7 @@ import abc
import typing
import netaddr
from oslo_concurrency import lockutils
import tobiko
from tobiko import config
@ -33,8 +34,11 @@ from tobiko.openstack.stacks import _nova
CONF = config.CONF
class VlanNetworkStackFixture(_neutron.NetworkStackFixture):
pass
class VlanNetworkStackFixture(_neutron.NetworkBaseStackFixture):
@lockutils.synchronized(
'create_vlan_network_stack', external=True, lock_path=tobiko.LOCK_DIR)
def setup_stack(self):
super().setup_stack()
class VlanProxyServerStackFixture(_cirros.CirrosServerStackFixture):

9
tobiko/tests/functional/openstack/stacks/test_neutron.py
View File

@ -15,6 +15,7 @@
# under the License.
from __future__ import absolute_import
from oslo_concurrency import lockutils
from oslo_log import log
import testtools
@ -150,8 +151,12 @@ class RouterInterfaceTestRouter(stacks.RouterStackFixture):
pass
class RouterInterfaceTestNetwork(stacks.NetworkStackFixture):
pass
class RouterInterfaceTestNetwork(stacks.NetworkBaseStackFixture):
@lockutils.synchronized(
'create_router_interface_network_stack',
external=True, lock_path=tobiko.LOCK_DIR)
def setup_stack(self):
super().setup_stack()
@keystone.skip_unless_has_keystone_credentials()

10
tobiko/tests/scenario/neutron/test_router.py
View File

@ -20,6 +20,7 @@ import re
import typing
import pytest
from oslo_concurrency import lockutils
from oslo_log import log
import testtools
@ -170,7 +171,7 @@ class RouterNamespaceTestBase:
return self.network_stack.gateway_stack
@property
def network_stack(self) -> stacks.NetworkStackFixture:
def network_stack(self) -> stacks.NetworkBaseStackFixture:
return self.server_stack.network_stack
@property
@ -206,10 +207,15 @@ class DvrRouterStackFixture(stacks.RouterStackFixture):
distributed = True
class DvrNetworkStackFixture(stacks.NetworkStackFixture):
class DvrNetworkStackFixture(stacks.NetworkBaseStackFixture):
gateway_stack = tobiko.required_fixture(DvrRouterStackFixture,
setup=False)
@lockutils.synchronized(
'create_dvr_network_stack', external=True, lock_path=tobiko.LOCK_DIR)
def setup_stack(self):
super().setup_stack()
class DvrServerStackFixture(stacks.CirrosServerStackFixture):
network_stack = tobiko.required_fixture(DvrNetworkStackFixture,