diff --git a/cloudbaseinit/plugins/windows/userdata.py b/cloudbaseinit/plugins/windows/userdata.py index 1de0ed45..fdf77b36 100644 --- a/cloudbaseinit/plugins/windows/userdata.py +++ b/cloudbaseinit/plugins/windows/userdata.py @@ -145,6 +145,11 @@ class UserDataPlugin(base.BasePlugin): plugin_status = base.PLUGIN_EXECUTION_DONE reboot = False + try: + ret_val = int(ret_val) + except (ValueError, TypeError): + ret_val = 0 + if ret_val >= 1001 and ret_val <= 1003: reboot = bool(ret_val & 1) if ret_val & 2: @@ -153,5 +158,11 @@ class UserDataPlugin(base.BasePlugin): return (plugin_status, reboot) def _process_non_multi_part(self, user_data): - ret_val = userdatautils.execute_user_data_script(user_data) + if user_data.startswith('#cloud-config'): + user_data_plugins = factory.load_plugins() + cloud_config_plugin = user_data_plugins.get('text/cloud-config') + ret_val = cloud_config_plugin.process(user_data) + else: + ret_val = userdatautils.execute_user_data_script(user_data) + return self._get_plugin_return_value(ret_val) diff --git a/cloudbaseinit/plugins/windows/userdataplugins/cloudconfig.py b/cloudbaseinit/plugins/windows/userdataplugins/cloudconfig.py index a593b902..67fb8af0 100644 --- a/cloudbaseinit/plugins/windows/userdataplugins/cloudconfig.py +++ b/cloudbaseinit/plugins/windows/userdataplugins/cloudconfig.py @@ -13,16 +13,205 @@ # License for the specific language governing permissions and limitations # under the License. +import base64 +import gzip +import io +import os + +from oslo.config import cfg +import yaml + from cloudbaseinit.openstack.common import log as logging from cloudbaseinit.plugins.windows.userdataplugins import base + LOG = logging.getLogger(__name__) +OPTS = [ + cfg.ListOpt( + 'cloud_config_plugins', + default=[], + help=( + 'List which contains the name of the plugins ordered by priority.' + ), + ) +] +CONF = cfg.CONF +CONF.register_opts(OPTS) + +DEFAULT_MIME_TYPE = 'text/plain' +DEFAULT_PERMISSIONS = 0o644 +BASE64_MIME = 'application/base64' +GZIP_MIME = 'application/x-gzip' + + +def decode_steps(encoding): + """Predict the decoding steps required to obtain the initial content.""" + encoding = encoding.lower().strip() if encoding else '' + if encoding in ('gz', 'gzip'): + return [GZIP_MIME] + + if encoding in ('gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64'): + return [BASE64_MIME, GZIP_MIME] + + if encoding in ('b64', 'base64'): + return [BASE64_MIME] + + if encoding: + LOG.warning("Unknown encoding type %s, assuming %s", + encoding, DEFAULT_MIME_TYPE) + + return [DEFAULT_MIME_TYPE] + + +def process_permissions(permissions): + """Safe process the permissions value.""" + if type(permissions) in (int, float): + permissions = int(permissions) + else: + try: + permissions = int(permissions, 8) + except (ValueError, TypeError): + LOG.warning("Fail to process permissions %s, assuming %s", + permissions, DEFAULT_PERMISSIONS) + permissions = DEFAULT_PERMISSIONS + + return permissions + + +def process_content(content, encoding): + """Decode the content taking into consideration the encoding.""" + result = str(content) + for mime_type in decode_steps(encoding): + if mime_type == GZIP_MIME: + bufferio = io.BytesIO(content) + with gzip.GzipFile(fileobj=bufferio, mode='rb') as file_handle: + try: + result = file_handle.read() + except (IOError, ValueError) as exc: + LOG.exception( + "Fail to decompress gzip content. Exception: %s", exc) + elif mime_type == BASE64_MIME: + try: + result = base64.b64decode(result) + except (ValueError, TypeError) as exc: + LOG.exception( + "Fail to decode base64 content. Exception: %s", exc) + return result + + +def write_file(path, content, permissions=DEFAULT_PERMISSIONS, open_mode="wb"): + """Writes a file with the given content + + Also the function sets the file mode as specified. + The function arguments are the following: + path: The absolute path to the location on the filesystem where + the file should be written. + content: The content that should be placed in the file. + permissions:The octal permissions set that should be given for + this file. + open_mode: The open mode used when opening the file. + """ + dirname = os.path.dirname(path) + if not os.path.isdir(dirname): + try: + os.makedirs(dirname) + except OSError as exc: + LOG.exception(exc) + return False + + with open(path, open_mode) as file_handle: + file_handle.write(content) + file_handle.flush() + + os.chmod(path, permissions) + return True class CloudConfigPlugin(base.BaseUserDataPlugin): + def __init__(self): super(CloudConfigPlugin, self).__init__("text/cloud-config") + self._plugins_order = CONF.cloud_config_plugins + + def _priority(self, plugin): + """Predict the priority for this plugin + + Returns a numeric value that represents the priority of the plugin + designated by the received key. + + Note: If the priority for a plugin is not specified, it will designate + the lowest priority for it. + """ + try: + return self._plugins_order.index(plugin) + except ValueError: + return len(self._plugins_order) + + def _content(self, part): + """Iterator over the deserialized information from the receivedpart.""" + loader = getattr(yaml, 'CLoader', yaml.Loader) + + try: + content = yaml.load(part, Loader=loader) + except ValueError: + LOG.error("Invalid yaml stream provided.") + return False + + if not isinstance(content, dict): + LOG.warning("Unsupported content type %s", type(content)) + return False + + # Create a list that will contain the information received in the order + # specified by the user. + return sorted(content.items(), + key=lambda item: self._priority(item[0])) + + def plugin_write_files(self, files): + """Plugin for writing files on the filesystem + + Receives a list of files in order to write them on disk. + Each file that should be written is represented by a dictionary which + can contain the following keys: + path: The absolute path to the location on the filesystem where + the file should be written. + content: The content that should be placed in the file. + owner: The user account and group that should be given ownership of + the file. + permissions: The octal permissions set that should be given for + this file. + encoding: An optional encoding specification for the file. + + Note: The only required keys in this dictionary are `path` and + `content`. + """ + + for current_file in files: + incomplete = False + for required_key in ('path', 'content'): + if required_key not in current_file: + incomplete = True + break + if incomplete: + LOG.warning("Missing required keys from file information %s", + current_file) + continue + + path = os.path.abspath(current_file['path']) + content = process_content(current_file['content'], + current_file.get('encoding')) + permissions = process_permissions(current_file.get('permissions')) + write_file(path, content, permissions) def process(self, part): - LOG.info("%s content is currently not supported" % - self.get_mime_type()) + content = self._content(part) or [] + for key, value in content: + method_name = "plugin_%s" % key.replace("-", "_") + method = getattr(self, method_name, None) + if not method: + LOG.info("Plugin %s is currently not supported", key) + continue + + try: + method(value) + except Exception as exc: + LOG.exception(exc) diff --git a/cloudbaseinit/tests/plugins/windows/test_userdata.py b/cloudbaseinit/tests/plugins/windows/test_userdata.py index 7c440f3f..e92ac1e2 100644 --- a/cloudbaseinit/tests/plugins/windows/test_userdata.py +++ b/cloudbaseinit/tests/plugins/windows/test_userdata.py @@ -256,3 +256,23 @@ class UserDataPluginTest(unittest.TestCase): mock_get_plugin_return_value.assert_called_once_with( mock_execute_user_data_script()) self.assertEqual(mock_get_plugin_return_value.return_value, response) + + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.factory.' + 'load_plugins') + @mock.patch('cloudbaseinit.plugins.windows.userdata.UserDataPlugin' + '._get_plugin_return_value') + def test_process_non_multi_part_cloud_config( + self, mock_get_plugin_return_value, mock_load_plugins): + user_data = '#cloud-config' + mock_return_value = mock.sentinel.return_value + mock_cloud_config_plugin = mock.Mock() + mock_cloud_config_plugin.process.return_value = mock_return_value + mock_get_plugin_return_value.return_value = mock_return_value + mock_load_plugins.return_value = { + 'text/cloud-config': mock_cloud_config_plugin} + return_value = self._userdata._process_non_multi_part( + user_data=user_data) + + mock_load_plugins.assert_called_once_with() + mock_cloud_config_plugin.process.assert_called_once_with(user_data) + self.assertEqual(mock_return_value, return_value) diff --git a/cloudbaseinit/tests/plugins/windows/userdataplugins/test_cloudconfig.py b/cloudbaseinit/tests/plugins/windows/userdataplugins/test_cloudconfig.py index 33e8afcc..cc4e7804 100644 --- a/cloudbaseinit/tests/plugins/windows/userdataplugins/test_cloudconfig.py +++ b/cloudbaseinit/tests/plugins/windows/userdataplugins/test_cloudconfig.py @@ -14,24 +14,222 @@ # License for the specific language governing permissions and limitations # under the License. +import importlib import mock import unittest from oslo.config import cfg -from cloudbaseinit.plugins.windows.userdataplugins import cloudconfig - CONF = cfg.CONF class CloudConfigPluginTests(unittest.TestCase): def setUp(self): - self._cloudconfig = cloudconfig.CloudConfigPlugin() + self._yaml_mock = mock.MagicMock() + self._module_patcher = mock.patch.dict( + 'sys.modules', {'yaml': self._yaml_mock}) + self._module_patcher.start() + self.cloudconfig = importlib.import_module( + 'cloudbaseinit.plugins.windows.userdataplugins.cloudconfig') + self._cloudconfig = self.cloudconfig.CloudConfigPlugin() + + def tearDown(self): + self._module_patcher.stop() + + def test_decode_steps(self): + expected_return_value = [ + [self.cloudconfig.GZIP_MIME], [self.cloudconfig.BASE64_MIME], + [self.cloudconfig.BASE64_MIME, self.cloudconfig.GZIP_MIME], + [self.cloudconfig.DEFAULT_MIME_TYPE], + [self.cloudconfig.DEFAULT_MIME_TYPE] + ] + return_value = [self.cloudconfig.decode_steps(encoding) + for encoding in ('gz', 'b64', 'gz+b64', 'fake', '')] + + self.assertEqual(expected_return_value, return_value) + + def test_process_permissions(self): + for permissions in (0o644, '0644', '0o644', 420, 420.1): + self.assertEqual( + 420, self.cloudconfig.process_permissions(permissions)) + + response = self.cloudconfig.process_permissions(mock.sentinel.invalid) + self.assertEqual(self.cloudconfig.DEFAULT_PERMISSIONS, response) + + def test_priority(self): + in_list = mock.sentinel.in_list + not_in_list = mock.sentinel.not_in_list + self._cloudconfig._plugins_order = [in_list] + + self.assertEqual(0, self._cloudconfig._priority(in_list)) + self.assertEqual(1, self._cloudconfig._priority(not_in_list)) + + @mock.patch('yaml.load') + def test_content(self, mock_yaml_load): + mock_yaml_load.side_effect = [ + ValueError("Invalid yaml stream provided."), + mock.sentinel.not_dict, + {} + ] + + for expected_return_value in (False, False, []): + return_value = self._cloudconfig._content(mock.sentinel.part) + self.assertEqual(expected_return_value, return_value) + + self.assertEqual(3, mock_yaml_load.call_count) + + @mock.patch('base64.b64decode') + @mock.patch('gzip.GzipFile.read') + @mock.patch('io.BytesIO') + def test_process_content(self, mock_bytes_io, mock_gzip_file, + mock_b64decode): + content = mock.sentinel.content + mock_gzip_file.return_value = content + mock_b64decode.return_value = content + + for encoding in ('gz', 'b64', 'gz+b64'): + return_value = self.cloudconfig.process_content(content, encoding) + self.assertEqual(content, return_value) + + self.assertEqual(2, mock_bytes_io.call_count) + self.assertEqual(2, mock_gzip_file.call_count) + self.assertEqual(2, mock_b64decode.call_count) + + @mock.patch('base64.b64decode') + @mock.patch('gzip.GzipFile.read') + @mock.patch('io.BytesIO') + def test_process_content_fail(self, mock_bytes_io, mock_gzip_file, + mock_b64decode): + content = mock.sentinel.content + mock_gzip_file.side_effect = [IOError(), ValueError()] + mock_b64decode.side_effect = [ValueError(), TypeError()] + + for encoding in ('gz', 'b64', 'gz+b64'): + return_value = self.cloudconfig.process_content(content, encoding) + self.assertEqual(str(content), return_value) + + self.assertEqual(2, mock_bytes_io.call_count) + self.assertEqual(2, mock_gzip_file.call_count) + self.assertEqual(2, mock_b64decode.call_count) + + @mock.patch('os.path.dirname') + @mock.patch('os.path.isdir') + @mock.patch('os.makedirs') + @mock.patch('os.chmod') + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.process_permissions') + def test_write_files(self, mock_process_permissions, + mock_chmod, mock_makedires, mock_isdir, + mock_dirname): + + path = mock.sentinel.path + permissions = mock.sentinel.permissions + content = mock.sentinel.content + open_mode = mock.sentinel.open_mode + mock_dirname.return_value = mock.sentinel.dirname + mock_isdir.return_value = False + + with mock.patch('cloudbaseinit.plugins.windows.userdataplugins.' + 'cloudconfig.open', mock.mock_open(), create=True): + self.cloudconfig.write_file(path, content, permissions, open_mode) + + mock_dirname.assert_called_once_with(path) + mock_isdir.assert_called_once_with(mock.sentinel.dirname) + mock_makedires.assert_called_once_with(mock.sentinel.dirname) + mock_chmod.assert_called_once_with(path, permissions) + + @mock.patch('os.path.dirname') + @mock.patch('os.path.isdir') + @mock.patch('os.makedirs') + def test_write_files_fail(self, mock_makedires, mock_isdir, + mock_dirname): + path = mock.sentinel.path + permissions = mock.sentinel.permissions + content = mock.sentinel.content + open_mode = mock.sentinel.open_mode + + mock_dirname.return_value = mock.sentinel.dirname + mock_isdir.return_value = False + mock_makedires.side_effect = [OSError()] + + return_value = self.cloudconfig.write_file(path, content, permissions, + open_mode) + mock_dirname.assert_called_once_with(path) + mock_isdir.assert_called_once_with(mock.sentinel.dirname) + mock_makedires.assert_called_once_with(mock.sentinel.dirname) + self.assertFalse(return_value) + + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.process_content') + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.process_permissions') + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.write_file') + @mock.patch('os.path.abspath') + def test_plugin_write_files(self, mock_abspath, mock_write_file, + mock_process_permissions, + mock_process_content): + path = mock.sentinel.path + content = mock.sentinel.content + permissions = mock.sentinel.permissions + files = [{'path': path, 'content': content}] + + mock_abspath.return_value = path + mock_process_permissions.return_value = permissions + mock_process_content.return_value = content + + self._cloudconfig.plugin_write_files(files) + mock_abspath.assert_called_once_with(path) + mock_process_permissions.assert_called_once_with(None) + mock_process_content.assert_called_once_with(content, None) + mock_write_file.assert_called_once_with(path, content, permissions) + + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.process_content') + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.process_permissions') + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.write_file') + @mock.patch('os.path.abspath') + def test_plugin_write_files_fail(self, mock_abspath, mock_write_file, + mock_process_permissions, + mock_process_content): + + self._cloudconfig.plugin_write_files([{}]) + self.assertEqual(0, mock_abspath.call_count) + self.assertEqual(0, mock_process_permissions.call_count) + self.assertEqual(0, mock_process_content.call_count) + self.assertEqual(0, mock_write_file.call_count) + + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.CloudConfigPlugin.plugin_write_files') + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.CloudConfigPlugin._content') + def test_process(self, mock_content, mock_plugin_write_files): + mock_content.side_effect = [ + [("write_files", mock.sentinel.content)], + [("invalid_plugin", mock.sentinel.content)] + ] + mock_part = mock.sentinel.part - @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.base' - '.BaseUserDataPlugin.get_mime_type') - def test_process(self, mock_get_mime_type): - mock_part = mock.MagicMock() self._cloudconfig.process(mock_part) - mock_get_mime_type.assert_called_once_with() + mock_plugin_write_files.assert_called_once_with(mock.sentinel.content) + mock_content.assert_called_once_with(mock_part) + + mock_plugin_write_files.reset_mock() + self._cloudconfig.process(mock_part) + self.assertEqual(0, mock_plugin_write_files.call_count) + + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.CloudConfigPlugin.plugin_write_files') + @mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig' + '.CloudConfigPlugin._content') + def test_process_fail(self, mock_content, mock_plugin_write_files): + mock_content.return_value = [("write_files", mock.sentinel.content)] + mock_plugin_write_files.side_effect = [ValueError()] + mock_part = mock.sentinel.part + + self._cloudconfig.process(mock_part) + mock_content.assert_called_once_with(mock_part) + mock_plugin_write_files.assert_called_once_with(mock.sentinel.content) diff --git a/requirements.txt b/requirements.txt index dfd19515..e33f9a49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ six>=1.7.0 Babel>=1.3 oauth netifaces +PyYAML \ No newline at end of file