Add support for yaml user-data

CloudConfigPlugin receives information serialized into YAML format.
The information is deserialized and for each key value pair a
specialized plugin is called for processing this kind of task.

Implements-Blueprint: yaml-userdata

Change-Id: I273bc28415bc7fb37b2ef426868250ad152faec1

cloudbaseinit/plugins/windows/userdata.py

@@ -145,6 +145,11 @@ class UserDataPlugin(base.BasePlugin):
         plugin_status = base.PLUGIN_EXECUTION_DONE
         reboot = False
 
+        try:
+            ret_val = int(ret_val)
+        except (ValueError, TypeError):
+            ret_val = 0
+
         if ret_val >= 1001 and ret_val <= 1003:
             reboot = bool(ret_val & 1)
             if ret_val & 2:
@@ -153,5 +158,11 @@ class UserDataPlugin(base.BasePlugin):
         return (plugin_status, reboot)
 
     def _process_non_multi_part(self, user_data):
+        if user_data.startswith('#cloud-config'):
+            user_data_plugins = factory.load_plugins()
+            cloud_config_plugin = user_data_plugins.get('text/cloud-config')
+            ret_val = cloud_config_plugin.process(user_data)
+        else:
             ret_val = userdatautils.execute_user_data_script(user_data)
+
         return self._get_plugin_return_value(ret_val)

cloudbaseinit/plugins/windows/userdataplugins/cloudconfig.py

@@ -13,16 +13,205 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import base64
+import gzip
+import io
+import os
+
+from oslo.config import cfg
+import yaml
+
 from cloudbaseinit.openstack.common import log as logging
 from cloudbaseinit.plugins.windows.userdataplugins import base
 
+
 LOG = logging.getLogger(__name__)
+OPTS = [
+    cfg.ListOpt(
+        'cloud_config_plugins',
+        default=[],
+        help=(
+            'List which contains the name of the plugins ordered by priority.'
+        ),
+    )
+]
+CONF = cfg.CONF
+CONF.register_opts(OPTS)
+
+DEFAULT_MIME_TYPE = 'text/plain'
+DEFAULT_PERMISSIONS = 0o644
+BASE64_MIME = 'application/base64'
+GZIP_MIME = 'application/x-gzip'
+
+
+def decode_steps(encoding):
+    """Predict the decoding steps required to obtain the initial content."""
+    encoding = encoding.lower().strip() if encoding else ''
+    if encoding in ('gz', 'gzip'):
+        return [GZIP_MIME]
+
+    if encoding in ('gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64'):
+        return [BASE64_MIME, GZIP_MIME]
+
+    if encoding in ('b64', 'base64'):
+        return [BASE64_MIME]
+
+    if encoding:
+        LOG.warning("Unknown encoding type %s, assuming %s",
+                    encoding, DEFAULT_MIME_TYPE)
+
+    return [DEFAULT_MIME_TYPE]
+
+
+def process_permissions(permissions):
+    """Safe process the permissions value."""
+    if type(permissions) in (int, float):
+        permissions = int(permissions)
+    else:
+        try:
+            permissions = int(permissions, 8)
+        except (ValueError, TypeError):
+            LOG.warning("Fail to process permissions %s, assuming %s",
+                        permissions, DEFAULT_PERMISSIONS)
+            permissions = DEFAULT_PERMISSIONS
+
+    return permissions
+
+
+def process_content(content, encoding):
+    """Decode the content taking into consideration the encoding."""
+    result = str(content)
+    for mime_type in decode_steps(encoding):
+        if mime_type == GZIP_MIME:
+            bufferio = io.BytesIO(content)
+            with gzip.GzipFile(fileobj=bufferio, mode='rb') as file_handle:
+                try:
+                    result = file_handle.read()
+                except (IOError, ValueError) as exc:
+                    LOG.exception(
+                        "Fail to decompress gzip content. Exception: %s", exc)
+        elif mime_type == BASE64_MIME:
+            try:
+                result = base64.b64decode(result)
+            except (ValueError, TypeError) as exc:
+                LOG.exception(
+                    "Fail to decode base64 content. Exception: %s", exc)
+    return result
+
+
+def write_file(path, content, permissions=DEFAULT_PERMISSIONS, open_mode="wb"):
+    """Writes a file with the given content
+
+    Also the function sets the file mode as specified.
+    The function arguments are the following:
+        path: The absolute path to the location on the filesystem where
+        the file should be written.
+        content: The content that should be placed in the file.
+        permissions:The octal permissions set that should be given for
+        this file.
+        open_mode: The open mode used when opening the file.
+    """
+    dirname = os.path.dirname(path)
+    if not os.path.isdir(dirname):
+        try:
+            os.makedirs(dirname)
+        except OSError as exc:
+            LOG.exception(exc)
+            return False
+
+    with open(path, open_mode) as file_handle:
+        file_handle.write(content)
+        file_handle.flush()
+
+    os.chmod(path, permissions)
+    return True
 
 
 class CloudConfigPlugin(base.BaseUserDataPlugin):
+
     def __init__(self):
         super(CloudConfigPlugin, self).__init__("text/cloud-config")
+        self._plugins_order = CONF.cloud_config_plugins
+
+    def _priority(self, plugin):
+        """Predict the priority for this plugin
+
+        Returns a numeric value that represents the priority of the plugin
+        designated by the received key.
+
+        Note: If the priority for a plugin is not specified, it will designate
+        the lowest priority for it.
+        """
+        try:
+            return self._plugins_order.index(plugin)
+        except ValueError:
+            return len(self._plugins_order)
+
+    def _content(self, part):
+        """Iterator over the deserialized information from the receivedpart."""
+        loader = getattr(yaml, 'CLoader', yaml.Loader)
+
+        try:
+            content = yaml.load(part, Loader=loader)
+        except ValueError:
+            LOG.error("Invalid yaml stream provided.")
+            return False
+
+        if not isinstance(content, dict):
+            LOG.warning("Unsupported content type %s", type(content))
+            return False
+
+        # Create a list that will contain the information received in the order
+        # specified by the user.
+        return sorted(content.items(),
+                      key=lambda item: self._priority(item[0]))
+
+    def plugin_write_files(self, files):
+        """Plugin for writing files on the filesystem
+
+        Receives a list of files in order to write them on disk.
+        Each file that should be written is represented by a dictionary which
+        can contain the following keys:
+            path: The absolute path to the location on the filesystem where
+            the file should be written.
+            content: The content that should be placed in the file.
+            owner: The user account and group that should be given ownership of
+            the file.
+            permissions: The octal permissions set that should be given for
+            this file.
+            encoding: An optional encoding specification for the file.
+
+        Note: The only required keys in this dictionary are `path` and
+        `content`.
+        """
+
+        for current_file in files:
+            incomplete = False
+            for required_key in ('path', 'content'):
+                if required_key not in current_file:
+                    incomplete = True
+                    break
+            if incomplete:
+                LOG.warning("Missing required keys from file information %s",
+                            current_file)
+                continue
+
+            path = os.path.abspath(current_file['path'])
+            content = process_content(current_file['content'],
+                                      current_file.get('encoding'))
+            permissions = process_permissions(current_file.get('permissions'))
+            write_file(path, content, permissions)
 
     def process(self, part):
-        LOG.info("%s content is currently not supported" %
+        content = self._content(part) or []
-                 self.get_mime_type())
+        for key, value in content:
+            method_name = "plugin_%s" % key.replace("-", "_")
+            method = getattr(self, method_name, None)
+            if not method:
+                LOG.info("Plugin %s is currently not supported", key)
+                continue
+
+            try:
+                method(value)
+            except Exception as exc:
+                LOG.exception(exc)

cloudbaseinit/tests/plugins/windows/test_userdata.py

@@ -256,3 +256,23 @@ class UserDataPluginTest(unittest.TestCase):
         mock_get_plugin_return_value.assert_called_once_with(
             mock_execute_user_data_script())
         self.assertEqual(mock_get_plugin_return_value.return_value, response)
+
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.factory.'
+                'load_plugins')
+    @mock.patch('cloudbaseinit.plugins.windows.userdata.UserDataPlugin'
+                '._get_plugin_return_value')
+    def test_process_non_multi_part_cloud_config(
+            self, mock_get_plugin_return_value, mock_load_plugins):
+        user_data = '#cloud-config'
+        mock_return_value = mock.sentinel.return_value
+        mock_cloud_config_plugin = mock.Mock()
+        mock_cloud_config_plugin.process.return_value = mock_return_value
+        mock_get_plugin_return_value.return_value = mock_return_value
+        mock_load_plugins.return_value = {
+            'text/cloud-config': mock_cloud_config_plugin}
+        return_value = self._userdata._process_non_multi_part(
+            user_data=user_data)
+
+        mock_load_plugins.assert_called_once_with()
+        mock_cloud_config_plugin.process.assert_called_once_with(user_data)
+        self.assertEqual(mock_return_value, return_value)

cloudbaseinit/tests/plugins/windows/userdataplugins/test_cloudconfig.py

@@ -14,24 +14,222 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import importlib
 import mock
 import unittest
 
 from oslo.config import cfg
 
-from cloudbaseinit.plugins.windows.userdataplugins import cloudconfig
-
 CONF = cfg.CONF
 
 
 class CloudConfigPluginTests(unittest.TestCase):
 
     def setUp(self):
-        self._cloudconfig = cloudconfig.CloudConfigPlugin()
+        self._yaml_mock = mock.MagicMock()
+        self._module_patcher = mock.patch.dict(
+            'sys.modules', {'yaml': self._yaml_mock})
+        self._module_patcher.start()
+        self.cloudconfig = importlib.import_module(
+            'cloudbaseinit.plugins.windows.userdataplugins.cloudconfig')
+        self._cloudconfig = self.cloudconfig.CloudConfigPlugin()
+
+    def tearDown(self):
+        self._module_patcher.stop()
+
+    def test_decode_steps(self):
+        expected_return_value = [
+            [self.cloudconfig.GZIP_MIME], [self.cloudconfig.BASE64_MIME],
+            [self.cloudconfig.BASE64_MIME, self.cloudconfig.GZIP_MIME],
+            [self.cloudconfig.DEFAULT_MIME_TYPE],
+            [self.cloudconfig.DEFAULT_MIME_TYPE]
+        ]
+        return_value = [self.cloudconfig.decode_steps(encoding)
+                        for encoding in ('gz', 'b64', 'gz+b64', 'fake', '')]
+
+        self.assertEqual(expected_return_value, return_value)
+
+    def test_process_permissions(self):
+        for permissions in (0o644, '0644', '0o644', 420, 420.1):
+            self.assertEqual(
+                420, self.cloudconfig.process_permissions(permissions))
+
+        response = self.cloudconfig.process_permissions(mock.sentinel.invalid)
+        self.assertEqual(self.cloudconfig.DEFAULT_PERMISSIONS, response)
+
+    def test_priority(self):
+        in_list = mock.sentinel.in_list
+        not_in_list = mock.sentinel.not_in_list
+        self._cloudconfig._plugins_order = [in_list]
+
+        self.assertEqual(0, self._cloudconfig._priority(in_list))
+        self.assertEqual(1, self._cloudconfig._priority(not_in_list))
+
+    @mock.patch('yaml.load')
+    def test_content(self, mock_yaml_load):
+        mock_yaml_load.side_effect = [
+            ValueError("Invalid yaml stream provided."),
+            mock.sentinel.not_dict,
+            {}
+        ]
+
+        for expected_return_value in (False, False, []):
+            return_value = self._cloudconfig._content(mock.sentinel.part)
+            self.assertEqual(expected_return_value, return_value)
+
+        self.assertEqual(3, mock_yaml_load.call_count)
+
+    @mock.patch('base64.b64decode')
+    @mock.patch('gzip.GzipFile.read')
+    @mock.patch('io.BytesIO')
+    def test_process_content(self, mock_bytes_io, mock_gzip_file,
+                             mock_b64decode):
+        content = mock.sentinel.content
+        mock_gzip_file.return_value = content
+        mock_b64decode.return_value = content
+
+        for encoding in ('gz', 'b64', 'gz+b64'):
+            return_value = self.cloudconfig.process_content(content, encoding)
+            self.assertEqual(content, return_value)
+
+        self.assertEqual(2, mock_bytes_io.call_count)
+        self.assertEqual(2, mock_gzip_file.call_count)
+        self.assertEqual(2, mock_b64decode.call_count)
+
+    @mock.patch('base64.b64decode')
+    @mock.patch('gzip.GzipFile.read')
+    @mock.patch('io.BytesIO')
+    def test_process_content_fail(self, mock_bytes_io, mock_gzip_file,
+                                  mock_b64decode):
+        content = mock.sentinel.content
+        mock_gzip_file.side_effect = [IOError(), ValueError()]
+        mock_b64decode.side_effect = [ValueError(), TypeError()]
+
+        for encoding in ('gz', 'b64', 'gz+b64'):
+            return_value = self.cloudconfig.process_content(content, encoding)
+            self.assertEqual(str(content), return_value)
+
+        self.assertEqual(2, mock_bytes_io.call_count)
+        self.assertEqual(2, mock_gzip_file.call_count)
+        self.assertEqual(2, mock_b64decode.call_count)
+
+    @mock.patch('os.path.dirname')
+    @mock.patch('os.path.isdir')
+    @mock.patch('os.makedirs')
+    @mock.patch('os.chmod')
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.process_permissions')
+    def test_write_files(self, mock_process_permissions,
+                         mock_chmod, mock_makedires, mock_isdir,
+                         mock_dirname):
+
+        path = mock.sentinel.path
+        permissions = mock.sentinel.permissions
+        content = mock.sentinel.content
+        open_mode = mock.sentinel.open_mode
+        mock_dirname.return_value = mock.sentinel.dirname
+        mock_isdir.return_value = False
+
+        with mock.patch('cloudbaseinit.plugins.windows.userdataplugins.'
+                        'cloudconfig.open', mock.mock_open(), create=True):
+            self.cloudconfig.write_file(path, content, permissions, open_mode)
+
+        mock_dirname.assert_called_once_with(path)
+        mock_isdir.assert_called_once_with(mock.sentinel.dirname)
+        mock_makedires.assert_called_once_with(mock.sentinel.dirname)
+        mock_chmod.assert_called_once_with(path, permissions)
+
+    @mock.patch('os.path.dirname')
+    @mock.patch('os.path.isdir')
+    @mock.patch('os.makedirs')
+    def test_write_files_fail(self, mock_makedires, mock_isdir,
+                              mock_dirname):
+        path = mock.sentinel.path
+        permissions = mock.sentinel.permissions
+        content = mock.sentinel.content
+        open_mode = mock.sentinel.open_mode
+
+        mock_dirname.return_value = mock.sentinel.dirname
+        mock_isdir.return_value = False
+        mock_makedires.side_effect = [OSError()]
+
+        return_value = self.cloudconfig.write_file(path, content, permissions,
+                                                   open_mode)
+        mock_dirname.assert_called_once_with(path)
+        mock_isdir.assert_called_once_with(mock.sentinel.dirname)
+        mock_makedires.assert_called_once_with(mock.sentinel.dirname)
+        self.assertFalse(return_value)
+
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.process_content')
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.process_permissions')
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.write_file')
+    @mock.patch('os.path.abspath')
+    def test_plugin_write_files(self, mock_abspath, mock_write_file,
+                                mock_process_permissions,
+                                mock_process_content):
+        path = mock.sentinel.path
+        content = mock.sentinel.content
+        permissions = mock.sentinel.permissions
+        files = [{'path': path, 'content': content}]
+
+        mock_abspath.return_value = path
+        mock_process_permissions.return_value = permissions
+        mock_process_content.return_value = content
+
+        self._cloudconfig.plugin_write_files(files)
+        mock_abspath.assert_called_once_with(path)
+        mock_process_permissions.assert_called_once_with(None)
+        mock_process_content.assert_called_once_with(content, None)
+        mock_write_file.assert_called_once_with(path, content, permissions)
+
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.process_content')
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.process_permissions')
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.write_file')
+    @mock.patch('os.path.abspath')
+    def test_plugin_write_files_fail(self, mock_abspath, mock_write_file,
+                                     mock_process_permissions,
+                                     mock_process_content):
+
+        self._cloudconfig.plugin_write_files([{}])
+        self.assertEqual(0, mock_abspath.call_count)
+        self.assertEqual(0, mock_process_permissions.call_count)
+        self.assertEqual(0, mock_process_content.call_count)
+        self.assertEqual(0, mock_write_file.call_count)
+
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.CloudConfigPlugin.plugin_write_files')
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.CloudConfigPlugin._content')
+    def test_process(self, mock_content, mock_plugin_write_files):
+        mock_content.side_effect = [
+            [("write_files", mock.sentinel.content)],
+            [("invalid_plugin", mock.sentinel.content)]
+        ]
+        mock_part = mock.sentinel.part
 
-    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.base'
-                '.BaseUserDataPlugin.get_mime_type')
-    def test_process(self, mock_get_mime_type):
-        mock_part = mock.MagicMock()
         self._cloudconfig.process(mock_part)
-        mock_get_mime_type.assert_called_once_with()
+        mock_plugin_write_files.assert_called_once_with(mock.sentinel.content)
+        mock_content.assert_called_once_with(mock_part)
+
+        mock_plugin_write_files.reset_mock()
+        self._cloudconfig.process(mock_part)
+        self.assertEqual(0, mock_plugin_write_files.call_count)
+
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.CloudConfigPlugin.plugin_write_files')
+    @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
+                '.CloudConfigPlugin._content')
+    def test_process_fail(self, mock_content, mock_plugin_write_files):
+        mock_content.return_value = [("write_files", mock.sentinel.content)]
+        mock_plugin_write_files.side_effect = [ValueError()]
+        mock_part = mock.sentinel.part
+
+        self._cloudconfig.process(mock_part)
+        mock_content.assert_called_once_with(mock_part)
+        mock_plugin_write_files.assert_called_once_with(mock.sentinel.content)

requirements.txt

@@ -8,3 +8,4 @@ six>=1.7.0
 Babel>=1.3
 oauth
 netifaces
+PyYAML