Add APIs to manage cloud-init on Nova servers

Change-Id: Id705e465e44f18d4155cff8d46f26b94cd651801

tobiko/openstack/nova/__init__.py

@@ -14,6 +14,7 @@
 from __future__ import absolute_import
 
 from tobiko.openstack.nova import _client
+from tobiko.openstack.nova import _cloud_init
 from tobiko.openstack.nova import _hypervisor
 from tobiko.openstack.nova import _server
 
@@ -34,6 +35,13 @@ ServerStatusTimeout = _client.ServerStatusTimeout
 shutoff_server = _client.shutoff_server
 activate_server = _client.activate_server
 
+WaitForCloudInitTimeoutError = _cloud_init.WaitForCloudInitTimeoutError
+cloud_config = _cloud_init.cloud_config
+get_cloud_init_status = _cloud_init.get_cloud_init_status
+user_data = _cloud_init.user_data
+wait_for_cloud_init_done = _cloud_init.wait_for_cloud_init_done
+wait_for_cloud_init_status = _cloud_init.wait_for_cloud_init_status
+
 skip_if_missing_hypervisors = _hypervisor.skip_if_missing_hypervisors
 get_same_host_hypervisors = _hypervisor.get_same_host_hypervisors
 get_different_host_hypervisors = _hypervisor.get_different_host_hypervisors

tobiko/openstack/nova/_cloud_init.py

@@ -0,0 +1,124 @@
+# Copyright 2020 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import collections
+import json
+import time
+
+from oslo_log import log
+import yaml
+
+import tobiko
+from tobiko.shell import sh
+
+LOG = log.getLogger(__name__)
+
+
+def user_data(*args, **kwargs):
+    config = cloud_config(*args, **kwargs)
+    if config:
+        return config.user_data
+    else:
+        return ''
+
+
+def cloud_config(*args, **kwargs):
+    return combine_cloud_configs(args + (kwargs,))
+
+
+def combine_cloud_configs(objs):
+    packages = []
+    runcmd = []
+    for obj in objs:
+        if obj:
+            if not isinstance(obj, collections.abc.Mapping):
+                obj = dict(obj)
+            for package in obj.pop('packages', []):
+                if package and package not in packages:
+                    packages.append(package)
+            for cmdline in obj.pop('runcmd', []):
+                if cmdline:
+                    cmdline = list(sh.shell_command(cmdline))
+                    if cmdline:
+                        runcmd.append(cmdline)
+            if obj:
+                message = ('Invalid cloud-init parameters:\n' +
+                           json.dumps(obj, indent=4, sort_keys=True))
+                raise ValueError(message)
+
+    return CloudConfig.create(packages=packages or None,
+                              runcmd=runcmd or None)
+
+
+class CloudConfig(dict):
+
+    @classmethod
+    def create(cls, **kwargs):
+        return cls((k, v)
+                   for k, v in kwargs.items()
+                   if v is not None)
+
+    @property
+    def user_data(self):
+        return '#cloud-config\n' + yaml.dump(dict(self))
+
+    def __add__(self, other):
+        return combine_cloud_configs([self, other])
+
+
+class WaitForCloudInitTimeoutError(tobiko.TobikoException):
+    message = ("after {enlapsed_time} seconds cloud-init status of host "
+               "{hostname!r} is still {actual!r} while it is expecting to "
+               "be in {expected!r}")
+
+
+def get_cloud_init_status(ssh_client=None, timeout=None):
+    output = sh.execute('cloud-init status',
+                        ssh_client=ssh_client,
+                        timeout=timeout,
+                        sudo=True).stdout
+    return yaml.load(output)['status']
+
+
+def wait_for_cloud_init_done(ssh_client=None, timeout=None,
+                             sleep_interval=None):
+    return wait_for_cloud_init_status(expected={'done'},
+                                      ssh_client=ssh_client,
+                                      timeout=timeout,
+                                      sleep_interval=sleep_interval)
+
+
+def wait_for_cloud_init_status(expected, ssh_client=None, timeout=None,
+                               sleep_interval=None):
+    expected = set(expected)
+    timeout = timeout and float(timeout) or 600.
+    sleep_interval = sleep_interval and float(sleep_interval) or 5.
+    start_time = time.time()
+    actual = get_cloud_init_status(ssh_client=ssh_client, timeout=timeout)
+    while actual not in expected:
+        enlapsed_time = time.time() - start_time
+        if enlapsed_time >= timeout:
+            raise WaitForCloudInitTimeoutError(hostname=ssh_client.hostname,
+                                               actual=actual,
+                                               expected=expected,
+                                               enlapsed_time=enlapsed_time)
+
+        LOG.debug("Waiting cloud-init status on host %r to switch from %r to "
+                  "%r...",
+                  ssh_client.hostname, actual, expected)
+        time.sleep(sleep_interval)
+        actual = get_cloud_init_status(ssh_client=ssh_client,
+                                       timeout=timeout-enlapsed_time)
+    return actual

tobiko/openstack/stacks/_nova.py

@@ -251,6 +251,12 @@ class ServerStackFixture(heat.HeatStackFixture):
         return nova.get_console_output(server=self.server_id,
                                        length=self.max_console_output_length)
 
+    cloud_config = nova.cloud_config()
+
+    @property
+    def user_data(self):
+        return nova.user_data(self.cloud_config)
+
 
 class PeerServerStackFixture(ServerStackFixture):
     """Server witch networking access requires passing by a peer Nova server

tobiko/tests/unit/openstack/nova/test_cloud_init.py

@@ -0,0 +1,71 @@
+# Copyright 2020 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import yaml
+
+import testtools
+
+from tobiko.openstack import nova
+
+
+class TestUserData(testtools.TestCase):
+
+    def test_user_data(self):
+        self.assertEqual('', nova.user_data())
+
+    def test_user_data_with_packages(self):
+        user_data = nova.user_data({'packages': [1, 2]}, packages=[2, 3])
+        self.assert_equal_cloud_config({"packages": [1, 2, 3]},
+                                       user_data)
+
+    def test_user_data_with_runcmd(self):
+        user_data = nova.user_data({'runcmd': [["echo", 1]]},
+                                   runcmd=['echo 2'])
+        self.assert_equal_cloud_config({'runcmd': [['echo', '1'],
+                                                   ['echo', '2']]},
+                                       user_data)
+
+    def test_user_data_with_invalid(self):
+        ex = self.assertRaises(ValueError, nova.user_data, wrong='mistake')
+        self.assertEqual(
+            'Invalid cloud-init parameters:\n{\n    "wrong": "mistake"\n}',
+            str(ex))
+
+    def assert_equal_cloud_config(self, expected, actual):
+        self.assertTrue(actual.startswith('#cloud-config'))
+        self.assertEqual(expected, yaml.load(actual))
+
+
+class TestCloudConfig(testtools.TestCase):
+
+    def test_cloud_config(self):
+        self.assertEqual({}, nova.cloud_config())
+
+    def test_cloud_config_with_packages(self):
+        cloud_config = nova.cloud_config({'packages': [1, 2]}, packages=[2, 3])
+        self.assertEqual({"packages": [1, 2, 3]}, cloud_config)
+
+    def test_cloud_config_with_runcmd(self):
+        cloud_config = nova.cloud_config({'runcmd': [["echo", 1]]},
+                                         runcmd=['echo 2'])
+        self.assertEqual({'runcmd': [['echo', '1'],
+                                     ['echo', '2']]},
+                         cloud_config)
+
+    def test_cloud_config_with_invalid(self):
+        ex = self.assertRaises(ValueError, nova.cloud_config, wrong='mistake')
+        self.assertEqual(
+            'Invalid cloud-init parameters:\n{\n    "wrong": "mistake"\n}',
+            str(ex))