diff --git a/tobiko/openstack/nova/__init__.py b/tobiko/openstack/nova/__init__.py index f9a765f0e..0ab197bd9 100644 --- a/tobiko/openstack/nova/__init__.py +++ b/tobiko/openstack/nova/__init__.py @@ -38,6 +38,7 @@ list_services = _client.list_services nova_client = _client.nova_client NovaClientFixture = _client.NovaClientFixture wait_for_server_status = _client.wait_for_server_status +wait_for_guest_os_ready = _client.wait_for_guest_os_ready WaitForServerStatusError = _client.WaitForServerStatusError WaitForServerStatusTimeout = _client.WaitForServerStatusTimeout shutoff_server = _client.shutoff_server diff --git a/tobiko/openstack/nova/_client.py b/tobiko/openstack/nova/_client.py index 87ff02605..002d0eec9 100644 --- a/tobiko/openstack/nova/_client.py +++ b/tobiko/openstack/nova/_client.py @@ -314,6 +314,28 @@ def get_console_output(server: typing.Optional[ServerType] = None, return None +def wait_for_guest_os_ready(server: typing.Optional[ServerType] = None, + server_id: typing.Optional[str] = None, + timeout: tobiko.Seconds = 120., + interval: tobiko.Seconds = 5., + client: NovaClientType = None) -> None: + + def system_booted(): + console_output = get_console_output( + server=server, server_id=server_id, client=client) or '' + for line in console_output.splitlines(): + if 'login:' in line.lower(): + return True + return False + + for _ in tobiko.retry(timeout=timeout, interval=interval): + if system_booted(): + LOG.debug('VM boot completed successfully') + return + else: + LOG.debug('VM boot not completed yet') + + class HasNovaClientMixin(object): nova_client: NovaClientType = None diff --git a/tobiko/openstack/stacks/_nova.py b/tobiko/openstack/stacks/_nova.py index 56fed5116..f0e4ef0ac 100644 --- a/tobiko/openstack/stacks/_nova.py +++ b/tobiko/openstack/stacks/_nova.py @@ -20,6 +20,7 @@ import typing from abc import ABC import netaddr +from oslo_concurrency import lockutils from oslo_log import log import tobiko @@ -278,20 +279,24 @@ class ServerStackFixture(heat.HeatStackFixture, abc.ABC): hypervisors = nova.list_servers_hypervisors(self.same_host) if hypervisor not in hypervisors: self.migrate_server(live=True, - host=hypervisors.unique) + host=hypervisors.unique, + wait_for_guest_os=True) def validate_different_host_scheduler_hints(self, hypervisor): if self.different_host: hypervisors = nova.list_servers_hypervisors(self.different_host) if hypervisor in hypervisors: - self.migrate_server(live=True) + self.migrate_server(live=True, wait_for_guest_os=True) def migrate_server(self, live=False, host: str = None, - block_migration: bool = None) \ + block_migration: bool = None, + wait_for_guest_os: bool = False) \ -> nova.NovaServer: server = nova.activate_server(server=self.server_id) + if wait_for_guest_os: + nova.wait_for_guest_os_ready(server) if live: nova.live_migrate_server(server, host=host, @@ -424,6 +429,17 @@ class CloudInitServerStackFixture(ServerStackFixture, ABC): #: nax SWAP file size in bytes swap_maxsize: typing.Optional[int] = None + @lockutils.synchronized( + 'cloudinit_server_setup_fixture', + external=True, lock_path=tobiko.LOCK_DIR) + def setup_fixture(self): + super(CloudInitServerStackFixture, self).setup_fixture() + nova.wait_for_guest_os_ready(server_id=self.server_id, + timeout=900.) + if self.has_floating_ip: + self.assert_is_reachable() + self.wait_for_cloud_init_done() + @property def is_reachable_timeout(self) -> tobiko.Seconds: # I expect cloud-init based servers to be slow to boot diff --git a/tobiko/openstack/stacks/_ubuntu.py b/tobiko/openstack/stacks/_ubuntu.py index 3f765a3d6..f1bd03edb 100644 --- a/tobiko/openstack/stacks/_ubuntu.py +++ b/tobiko/openstack/stacks/_ubuntu.py @@ -41,7 +41,7 @@ class UbuntuMinimalImageFixture(glance.FileGlanceImageFixture): is_reachable_timeout = CONF.tobiko.nova.ubuntu_is_reachable_timeout @lockutils.synchronized( - 'ubuntu_minimal_setup_fixture', + 'ubuntu_minimal_image_setup_fixture', external=True, lock_path=tobiko.LOCK_DIR) def setup_fixture(self): super(UbuntuMinimalImageFixture, self).setup_fixture() diff --git a/tobiko/tests/scenario/nova/test_server.py b/tobiko/tests/scenario/nova/test_server.py index 86166f50b..17ab0287a 100644 --- a/tobiko/tests/scenario/nova/test_server.py +++ b/tobiko/tests/scenario/nova/test_server.py @@ -14,7 +14,6 @@ # under the License. from __future__ import absolute_import -import abc import contextlib import random @@ -188,18 +187,7 @@ class CirrosServerTest(BaseServerTest): stack = tobiko.required_fixture(CirrosServerStackFixture) -class CloudInitServerStackFixture(stacks.CloudInitServerStackFixture, - abc.ABC): - - def validate_created_stack(self): - stack = super().validate_created_stack() - self.wait_for_cloud_init_done() - self.assert_is_reachable() - return stack - - -class UbuntuMinimalServerStackFixture(CloudInitServerStackFixture, - stacks.UbuntuMinimalServerStackFixture): +class UbuntuMinimalServerStackFixture(stacks.UbuntuMinimalServerStackFixture): pass @@ -209,8 +197,7 @@ class UbuntuMinimalServerTest(BaseServerTest): stack = tobiko.required_fixture(UbuntuMinimalServerStackFixture) -class UbuntuServerStackFixture(CloudInitServerStackFixture, - stacks.UbuntuServerStackFixture): +class UbuntuServerStackFixture(stacks.UbuntuServerStackFixture): pass