From 417a8ab14b3977fc959d6359f8fa69f56d1fced3 Mon Sep 17 00:00:00 2001 From: Eduardo Olivares Date: Mon, 4 Sep 2023 17:48:01 +0200 Subject: [PATCH] Lock VM creation until guest OS is ready and cloud-init done Avoid concurrency issues by locking VM creation until it is completed. Add step that verifies that the VM instance boot is completed before releasing the lock and before the tests try to ping, ssh, etc that VM. Add another step that checks the VM FIP is pingable. Add a third step that checks the VM has completed the cloud-init procedure. Some tests (SameHostNetworkTest and DifferentHostNetworkTest) try to migrate VMs immediately after creating them in order to move them to the expected compute node. Sometimes, those VMs fail to contact the metadata service because their migration happens exactly when the guest OS was trying to obtain the metadata. With this patch, those VMs are not migrated until the guest OS is ready. Change-Id: Ia32b8c774e1c94277fa7243b7943a8b19763a501 --- tobiko/openstack/nova/__init__.py | 1 + tobiko/openstack/nova/_client.py | 22 ++++++++++++++++++++++ tobiko/openstack/stacks/_nova.py | 22 +++++++++++++++++++--- tobiko/openstack/stacks/_ubuntu.py | 2 +- tobiko/tests/scenario/nova/test_server.py | 17 ++--------------- 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/tobiko/openstack/nova/__init__.py b/tobiko/openstack/nova/__init__.py index f9a765f0e..0ab197bd9 100644 --- a/tobiko/openstack/nova/__init__.py +++ b/tobiko/openstack/nova/__init__.py @@ -38,6 +38,7 @@ list_services = _client.list_services nova_client = _client.nova_client NovaClientFixture = _client.NovaClientFixture wait_for_server_status = _client.wait_for_server_status +wait_for_guest_os_ready = _client.wait_for_guest_os_ready WaitForServerStatusError = _client.WaitForServerStatusError WaitForServerStatusTimeout = _client.WaitForServerStatusTimeout shutoff_server = _client.shutoff_server diff --git a/tobiko/openstack/nova/_client.py b/tobiko/openstack/nova/_client.py index 87ff02605..002d0eec9 100644 --- a/tobiko/openstack/nova/_client.py +++ b/tobiko/openstack/nova/_client.py @@ -314,6 +314,28 @@ def get_console_output(server: typing.Optional[ServerType] = None, return None +def wait_for_guest_os_ready(server: typing.Optional[ServerType] = None, + server_id: typing.Optional[str] = None, + timeout: tobiko.Seconds = 120., + interval: tobiko.Seconds = 5., + client: NovaClientType = None) -> None: + + def system_booted(): + console_output = get_console_output( + server=server, server_id=server_id, client=client) or '' + for line in console_output.splitlines(): + if 'login:' in line.lower(): + return True + return False + + for _ in tobiko.retry(timeout=timeout, interval=interval): + if system_booted(): + LOG.debug('VM boot completed successfully') + return + else: + LOG.debug('VM boot not completed yet') + + class HasNovaClientMixin(object): nova_client: NovaClientType = None diff --git a/tobiko/openstack/stacks/_nova.py b/tobiko/openstack/stacks/_nova.py index 56fed5116..f0e4ef0ac 100644 --- a/tobiko/openstack/stacks/_nova.py +++ b/tobiko/openstack/stacks/_nova.py @@ -20,6 +20,7 @@ import typing from abc import ABC import netaddr +from oslo_concurrency import lockutils from oslo_log import log import tobiko @@ -278,20 +279,24 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC): hypervisors = nova.list_servers_hypervisors(self.same_host) if hypervisor not in hypervisors: self.migrate_server(live=True, - host=hypervisors.unique) + host=hypervisors.unique, + wait_for_guest_os=True) def validate_different_host_scheduler_hints(self, hypervisor): if self.different_host: hypervisors = nova.list_servers_hypervisors(self.different_host) if hypervisor in hypervisors: - self.migrate_server(live=True) + self.migrate_server(live=True, wait_for_guest_os=True) def migrate_server(self, live=False, host: str = None, - block_migration: bool = None) \ + block_migration: bool = None, + wait_for_guest_os: bool = False) \ -> nova.NovaServer: server = nova.activate_server(server=self.server_id) + if wait_for_guest_os: + nova.wait_for_guest_os_ready(server) if live: nova.live_migrate_server(server, host=host, @@ -424,6 +429,17 @@ class CloudInitServerStackFixture(ServerStackFixture, ABC): #: nax SWAP file size in bytes swap_maxsize: typing.Optional[int] = None + @lockutils.synchronized( + 'cloudinit_server_setup_fixture', + external=True, lock_path=tobiko.LOCK_DIR) + def setup_fixture(self): + super(CloudInitServerStackFixture, self).setup_fixture() + nova.wait_for_guest_os_ready(server_id=self.server_id, + timeout=900.) + if self.has_floating_ip: + self.assert_is_reachable() + self.wait_for_cloud_init_done() + @property def is_reachable_timeout(self) -> tobiko.Seconds: # I expect cloud-init based servers to be slow to boot diff --git a/tobiko/openstack/stacks/_ubuntu.py b/tobiko/openstack/stacks/_ubuntu.py index 3f765a3d6..f1bd03edb 100644 --- a/tobiko/openstack/stacks/_ubuntu.py +++ b/tobiko/openstack/stacks/_ubuntu.py @@ -41,7 +41,7 @@ class UbuntuMinimalImageFixture(glance.FileGlanceImageFixture): is_reachable_timeout = CONF.tobiko.nova.ubuntu_is_reachable_timeout @lockutils.synchronized( - 'ubuntu_minimal_setup_fixture', + 'ubuntu_minimal_image_setup_fixture', external=True, lock_path=tobiko.LOCK_DIR) def setup_fixture(self): super(UbuntuMinimalImageFixture, self).setup_fixture() diff --git a/tobiko/tests/scenario/nova/test_server.py b/tobiko/tests/scenario/nova/test_server.py index 86166f50b..17ab0287a 100644 --- a/tobiko/tests/scenario/nova/test_server.py +++ b/tobiko/tests/scenario/nova/test_server.py @@ -14,7 +14,6 @@ # under the License. from __future__ import absolute_import -import abc import contextlib import random @@ -188,18 +187,7 @@ class CirrosServerTest(BaseServerTest): stack = tobiko.required_fixture(CirrosServerStackFixture) -class CloudInitServerStackFixture(stacks.CloudInitServerStackFixture, - abc.ABC): - - def validate_created_stack(self): - stack = super().validate_created_stack() - self.wait_for_cloud_init_done() - self.assert_is_reachable() - return stack - - -class UbuntuMinimalServerStackFixture(CloudInitServerStackFixture, - stacks.UbuntuMinimalServerStackFixture): +class UbuntuMinimalServerStackFixture(stacks.UbuntuMinimalServerStackFixture): pass @@ -209,8 +197,7 @@ class UbuntuMinimalServerTest(BaseServerTest): stack = tobiko.required_fixture(UbuntuMinimalServerStackFixture) -class UbuntuServerStackFixture(CloudInitServerStackFixture, - stacks.UbuntuServerStackFixture): +class UbuntuServerStackFixture(stacks.UbuntuServerStackFixture): pass