Add support for yaml user-data
CloudConfigPlugin receives information serialized into YAML format. The information is deserialized and for each key value pair a specialized plugin is called for processing this kind of task. Implements-Blueprint: yaml-userdata Change-Id: I273bc28415bc7fb37b2ef426868250ad152faec1
This commit is contained in:
Alexandru Coman
parent
93dae21ba7
commit
9b44bf99ed
@ -145,6 +145,11 @@ class UserDataPlugin(base.BasePlugin):
|
||||
plugin_status = base.PLUGIN_EXECUTION_DONE
|
||||
reboot = False
|
||||
|
||||
try:
|
||||
ret_val = int(ret_val)
|
||||
except (ValueError, TypeError):
|
||||
ret_val = 0
|
||||
|
||||
if ret_val >= 1001 and ret_val <= 1003:
|
||||
reboot = bool(ret_val & 1)
|
||||
if ret_val & 2:
|
||||
@ -153,5 +158,11 @@ class UserDataPlugin(base.BasePlugin):
|
||||
return (plugin_status, reboot)
|
||||
|
||||
def _process_non_multi_part(self, user_data):
|
||||
ret_val = userdatautils.execute_user_data_script(user_data)
|
||||
if user_data.startswith('#cloud-config'):
|
||||
user_data_plugins = factory.load_plugins()
|
||||
cloud_config_plugin = user_data_plugins.get('text/cloud-config')
|
||||
ret_val = cloud_config_plugin.process(user_data)
|
||||
else:
|
||||
ret_val = userdatautils.execute_user_data_script(user_data)
|
||||
|
||||
return self._get_plugin_return_value(ret_val)
|
||||
|
||||
@ -13,16 +13,205 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import base64
|
||||
import gzip
|
||||
import io
|
||||
import os
|
||||
|
||||
from oslo.config import cfg
|
||||
import yaml
|
||||
|
||||
from cloudbaseinit.openstack.common import log as logging
|
||||
from cloudbaseinit.plugins.windows.userdataplugins import base
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
OPTS = [
|
||||
cfg.ListOpt(
|
||||
'cloud_config_plugins',
|
||||
default=[],
|
||||
help=(
|
||||
'List which contains the name of the plugins ordered by priority.'
|
||||
),
|
||||
)
|
||||
]
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(OPTS)
|
||||
|
||||
DEFAULT_MIME_TYPE = 'text/plain'
|
||||
DEFAULT_PERMISSIONS = 0o644
|
||||
BASE64_MIME = 'application/base64'
|
||||
GZIP_MIME = 'application/x-gzip'
|
||||
|
||||
|
||||
def decode_steps(encoding):
|
||||
"""Predict the decoding steps required to obtain the initial content."""
|
||||
encoding = encoding.lower().strip() if encoding else ''
|
||||
if encoding in ('gz', 'gzip'):
|
||||
return [GZIP_MIME]
|
||||
|
||||
if encoding in ('gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64'):
|
||||
return [BASE64_MIME, GZIP_MIME]
|
||||
|
||||
if encoding in ('b64', 'base64'):
|
||||
return [BASE64_MIME]
|
||||
|
||||
if encoding:
|
||||
LOG.warning("Unknown encoding type %s, assuming %s",
|
||||
encoding, DEFAULT_MIME_TYPE)
|
||||
|
||||
return [DEFAULT_MIME_TYPE]
|
||||
|
||||
|
||||
def process_permissions(permissions):
|
||||
"""Safe process the permissions value."""
|
||||
if type(permissions) in (int, float):
|
||||
permissions = int(permissions)
|
||||
else:
|
||||
try:
|
||||
permissions = int(permissions, 8)
|
||||
except (ValueError, TypeError):
|
||||
LOG.warning("Fail to process permissions %s, assuming %s",
|
||||
permissions, DEFAULT_PERMISSIONS)
|
||||
permissions = DEFAULT_PERMISSIONS
|
||||
|
||||
return permissions
|
||||
|
||||
|
||||
def process_content(content, encoding):
|
||||
"""Decode the content taking into consideration the encoding."""
|
||||
result = str(content)
|
||||
for mime_type in decode_steps(encoding):
|
||||
if mime_type == GZIP_MIME:
|
||||
bufferio = io.BytesIO(content)
|
||||
with gzip.GzipFile(fileobj=bufferio, mode='rb') as file_handle:
|
||||
try:
|
||||
result = file_handle.read()
|
||||
except (IOError, ValueError) as exc:
|
||||
LOG.exception(
|
||||
"Fail to decompress gzip content. Exception: %s", exc)
|
||||
elif mime_type == BASE64_MIME:
|
||||
try:
|
||||
result = base64.b64decode(result)
|
||||
except (ValueError, TypeError) as exc:
|
||||
LOG.exception(
|
||||
"Fail to decode base64 content. Exception: %s", exc)
|
||||
return result
|
||||
|
||||
|
||||
def write_file(path, content, permissions=DEFAULT_PERMISSIONS, open_mode="wb"):
|
||||
"""Writes a file with the given content
|
||||
|
||||
Also the function sets the file mode as specified.
|
||||
The function arguments are the following:
|
||||
path: The absolute path to the location on the filesystem where
|
||||
the file should be written.
|
||||
content: The content that should be placed in the file.
|
||||
permissions:The octal permissions set that should be given for
|
||||
this file.
|
||||
open_mode: The open mode used when opening the file.
|
||||
"""
|
||||
dirname = os.path.dirname(path)
|
||||
if not os.path.isdir(dirname):
|
||||
try:
|
||||
os.makedirs(dirname)
|
||||
except OSError as exc:
|
||||
LOG.exception(exc)
|
||||
return False
|
||||
|
||||
with open(path, open_mode) as file_handle:
|
||||
file_handle.write(content)
|
||||
file_handle.flush()
|
||||
|
||||
os.chmod(path, permissions)
|
||||
return True
|
||||
|
||||
|
||||
class CloudConfigPlugin(base.BaseUserDataPlugin):
|
||||
|
||||
def __init__(self):
|
||||
super(CloudConfigPlugin, self).__init__("text/cloud-config")
|
||||
self._plugins_order = CONF.cloud_config_plugins
|
||||
|
||||
def _priority(self, plugin):
|
||||
"""Predict the priority for this plugin
|
||||
|
||||
Returns a numeric value that represents the priority of the plugin
|
||||
designated by the received key.
|
||||
|
||||
Note: If the priority for a plugin is not specified, it will designate
|
||||
the lowest priority for it.
|
||||
"""
|
||||
try:
|
||||
return self._plugins_order.index(plugin)
|
||||
except ValueError:
|
||||
return len(self._plugins_order)
|
||||
|
||||
def _content(self, part):
|
||||
"""Iterator over the deserialized information from the receivedpart."""
|
||||
loader = getattr(yaml, 'CLoader', yaml.Loader)
|
||||
|
||||
try:
|
||||
content = yaml.load(part, Loader=loader)
|
||||
except ValueError:
|
||||
LOG.error("Invalid yaml stream provided.")
|
||||
return False
|
||||
|
||||
if not isinstance(content, dict):
|
||||
LOG.warning("Unsupported content type %s", type(content))
|
||||
return False
|
||||
|
||||
# Create a list that will contain the information received in the order
|
||||
# specified by the user.
|
||||
return sorted(content.items(),
|
||||
key=lambda item: self._priority(item[0]))
|
||||
|
||||
def plugin_write_files(self, files):
|
||||
"""Plugin for writing files on the filesystem
|
||||
|
||||
Receives a list of files in order to write them on disk.
|
||||
Each file that should be written is represented by a dictionary which
|
||||
can contain the following keys:
|
||||
path: The absolute path to the location on the filesystem where
|
||||
the file should be written.
|
||||
content: The content that should be placed in the file.
|
||||
owner: The user account and group that should be given ownership of
|
||||
the file.
|
||||
permissions: The octal permissions set that should be given for
|
||||
this file.
|
||||
encoding: An optional encoding specification for the file.
|
||||
|
||||
Note: The only required keys in this dictionary are `path` and
|
||||
`content`.
|
||||
"""
|
||||
|
||||
for current_file in files:
|
||||
incomplete = False
|
||||
for required_key in ('path', 'content'):
|
||||
if required_key not in current_file:
|
||||
incomplete = True
|
||||
break
|
||||
if incomplete:
|
||||
LOG.warning("Missing required keys from file information %s",
|
||||
current_file)
|
||||
continue
|
||||
|
||||
path = os.path.abspath(current_file['path'])
|
||||
content = process_content(current_file['content'],
|
||||
current_file.get('encoding'))
|
||||
permissions = process_permissions(current_file.get('permissions'))
|
||||
write_file(path, content, permissions)
|
||||
|
||||
def process(self, part):
|
||||
LOG.info("%s content is currently not supported" %
|
||||
self.get_mime_type())
|
||||
content = self._content(part) or []
|
||||
for key, value in content:
|
||||
method_name = "plugin_%s" % key.replace("-", "_")
|
||||
method = getattr(self, method_name, None)
|
||||
if not method:
|
||||
LOG.info("Plugin %s is currently not supported", key)
|
||||
continue
|
||||
|
||||
try:
|
||||
method(value)
|
||||
except Exception as exc:
|
||||
LOG.exception(exc)
|
||||
|
||||
@ -256,3 +256,23 @@ class UserDataPluginTest(unittest.TestCase):
|
||||
mock_get_plugin_return_value.assert_called_once_with(
|
||||
mock_execute_user_data_script())
|
||||
self.assertEqual(mock_get_plugin_return_value.return_value, response)
|
||||
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.factory.'
|
||||
'load_plugins')
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdata.UserDataPlugin'
|
||||
'._get_plugin_return_value')
|
||||
def test_process_non_multi_part_cloud_config(
|
||||
self, mock_get_plugin_return_value, mock_load_plugins):
|
||||
user_data = '#cloud-config'
|
||||
mock_return_value = mock.sentinel.return_value
|
||||
mock_cloud_config_plugin = mock.Mock()
|
||||
mock_cloud_config_plugin.process.return_value = mock_return_value
|
||||
mock_get_plugin_return_value.return_value = mock_return_value
|
||||
mock_load_plugins.return_value = {
|
||||
'text/cloud-config': mock_cloud_config_plugin}
|
||||
return_value = self._userdata._process_non_multi_part(
|
||||
user_data=user_data)
|
||||
|
||||
mock_load_plugins.assert_called_once_with()
|
||||
mock_cloud_config_plugin.process.assert_called_once_with(user_data)
|
||||
self.assertEqual(mock_return_value, return_value)
|
||||
|
||||
@ -14,24 +14,222 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import importlib
|
||||
import mock
|
||||
import unittest
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from cloudbaseinit.plugins.windows.userdataplugins import cloudconfig
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class CloudConfigPluginTests(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self._cloudconfig = cloudconfig.CloudConfigPlugin()
|
||||
self._yaml_mock = mock.MagicMock()
|
||||
self._module_patcher = mock.patch.dict(
|
||||
'sys.modules', {'yaml': self._yaml_mock})
|
||||
self._module_patcher.start()
|
||||
self.cloudconfig = importlib.import_module(
|
||||
'cloudbaseinit.plugins.windows.userdataplugins.cloudconfig')
|
||||
self._cloudconfig = self.cloudconfig.CloudConfigPlugin()
|
||||
|
||||
def tearDown(self):
|
||||
self._module_patcher.stop()
|
||||
|
||||
def test_decode_steps(self):
|
||||
expected_return_value = [
|
||||
[self.cloudconfig.GZIP_MIME], [self.cloudconfig.BASE64_MIME],
|
||||
[self.cloudconfig.BASE64_MIME, self.cloudconfig.GZIP_MIME],
|
||||
[self.cloudconfig.DEFAULT_MIME_TYPE],
|
||||
[self.cloudconfig.DEFAULT_MIME_TYPE]
|
||||
]
|
||||
return_value = [self.cloudconfig.decode_steps(encoding)
|
||||
for encoding in ('gz', 'b64', 'gz+b64', 'fake', '')]
|
||||
|
||||
self.assertEqual(expected_return_value, return_value)
|
||||
|
||||
def test_process_permissions(self):
|
||||
for permissions in (0o644, '0644', '0o644', 420, 420.1):
|
||||
self.assertEqual(
|
||||
420, self.cloudconfig.process_permissions(permissions))
|
||||
|
||||
response = self.cloudconfig.process_permissions(mock.sentinel.invalid)
|
||||
self.assertEqual(self.cloudconfig.DEFAULT_PERMISSIONS, response)
|
||||
|
||||
def test_priority(self):
|
||||
in_list = mock.sentinel.in_list
|
||||
not_in_list = mock.sentinel.not_in_list
|
||||
self._cloudconfig._plugins_order = [in_list]
|
||||
|
||||
self.assertEqual(0, self._cloudconfig._priority(in_list))
|
||||
self.assertEqual(1, self._cloudconfig._priority(not_in_list))
|
||||
|
||||
@mock.patch('yaml.load')
|
||||
def test_content(self, mock_yaml_load):
|
||||
mock_yaml_load.side_effect = [
|
||||
ValueError("Invalid yaml stream provided."),
|
||||
mock.sentinel.not_dict,
|
||||
{}
|
||||
]
|
||||
|
||||
for expected_return_value in (False, False, []):
|
||||
return_value = self._cloudconfig._content(mock.sentinel.part)
|
||||
self.assertEqual(expected_return_value, return_value)
|
||||
|
||||
self.assertEqual(3, mock_yaml_load.call_count)
|
||||
|
||||
@mock.patch('base64.b64decode')
|
||||
@mock.patch('gzip.GzipFile.read')
|
||||
@mock.patch('io.BytesIO')
|
||||
def test_process_content(self, mock_bytes_io, mock_gzip_file,
|
||||
mock_b64decode):
|
||||
content = mock.sentinel.content
|
||||
mock_gzip_file.return_value = content
|
||||
mock_b64decode.return_value = content
|
||||
|
||||
for encoding in ('gz', 'b64', 'gz+b64'):
|
||||
return_value = self.cloudconfig.process_content(content, encoding)
|
||||
self.assertEqual(content, return_value)
|
||||
|
||||
self.assertEqual(2, mock_bytes_io.call_count)
|
||||
self.assertEqual(2, mock_gzip_file.call_count)
|
||||
self.assertEqual(2, mock_b64decode.call_count)
|
||||
|
||||
@mock.patch('base64.b64decode')
|
||||
@mock.patch('gzip.GzipFile.read')
|
||||
@mock.patch('io.BytesIO')
|
||||
def test_process_content_fail(self, mock_bytes_io, mock_gzip_file,
|
||||
mock_b64decode):
|
||||
content = mock.sentinel.content
|
||||
mock_gzip_file.side_effect = [IOError(), ValueError()]
|
||||
mock_b64decode.side_effect = [ValueError(), TypeError()]
|
||||
|
||||
for encoding in ('gz', 'b64', 'gz+b64'):
|
||||
return_value = self.cloudconfig.process_content(content, encoding)
|
||||
self.assertEqual(str(content), return_value)
|
||||
|
||||
self.assertEqual(2, mock_bytes_io.call_count)
|
||||
self.assertEqual(2, mock_gzip_file.call_count)
|
||||
self.assertEqual(2, mock_b64decode.call_count)
|
||||
|
||||
@mock.patch('os.path.dirname')
|
||||
@mock.patch('os.path.isdir')
|
||||
@mock.patch('os.makedirs')
|
||||
@mock.patch('os.chmod')
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.process_permissions')
|
||||
def test_write_files(self, mock_process_permissions,
|
||||
mock_chmod, mock_makedires, mock_isdir,
|
||||
mock_dirname):
|
||||
|
||||
path = mock.sentinel.path
|
||||
permissions = mock.sentinel.permissions
|
||||
content = mock.sentinel.content
|
||||
open_mode = mock.sentinel.open_mode
|
||||
mock_dirname.return_value = mock.sentinel.dirname
|
||||
mock_isdir.return_value = False
|
||||
|
||||
with mock.patch('cloudbaseinit.plugins.windows.userdataplugins.'
|
||||
'cloudconfig.open', mock.mock_open(), create=True):
|
||||
self.cloudconfig.write_file(path, content, permissions, open_mode)
|
||||
|
||||
mock_dirname.assert_called_once_with(path)
|
||||
mock_isdir.assert_called_once_with(mock.sentinel.dirname)
|
||||
mock_makedires.assert_called_once_with(mock.sentinel.dirname)
|
||||
mock_chmod.assert_called_once_with(path, permissions)
|
||||
|
||||
@mock.patch('os.path.dirname')
|
||||
@mock.patch('os.path.isdir')
|
||||
@mock.patch('os.makedirs')
|
||||
def test_write_files_fail(self, mock_makedires, mock_isdir,
|
||||
mock_dirname):
|
||||
path = mock.sentinel.path
|
||||
permissions = mock.sentinel.permissions
|
||||
content = mock.sentinel.content
|
||||
open_mode = mock.sentinel.open_mode
|
||||
|
||||
mock_dirname.return_value = mock.sentinel.dirname
|
||||
mock_isdir.return_value = False
|
||||
mock_makedires.side_effect = [OSError()]
|
||||
|
||||
return_value = self.cloudconfig.write_file(path, content, permissions,
|
||||
open_mode)
|
||||
mock_dirname.assert_called_once_with(path)
|
||||
mock_isdir.assert_called_once_with(mock.sentinel.dirname)
|
||||
mock_makedires.assert_called_once_with(mock.sentinel.dirname)
|
||||
self.assertFalse(return_value)
|
||||
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.process_content')
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.process_permissions')
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.write_file')
|
||||
@mock.patch('os.path.abspath')
|
||||
def test_plugin_write_files(self, mock_abspath, mock_write_file,
|
||||
mock_process_permissions,
|
||||
mock_process_content):
|
||||
path = mock.sentinel.path
|
||||
content = mock.sentinel.content
|
||||
permissions = mock.sentinel.permissions
|
||||
files = [{'path': path, 'content': content}]
|
||||
|
||||
mock_abspath.return_value = path
|
||||
mock_process_permissions.return_value = permissions
|
||||
mock_process_content.return_value = content
|
||||
|
||||
self._cloudconfig.plugin_write_files(files)
|
||||
mock_abspath.assert_called_once_with(path)
|
||||
mock_process_permissions.assert_called_once_with(None)
|
||||
mock_process_content.assert_called_once_with(content, None)
|
||||
mock_write_file.assert_called_once_with(path, content, permissions)
|
||||
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.process_content')
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.process_permissions')
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.write_file')
|
||||
@mock.patch('os.path.abspath')
|
||||
def test_plugin_write_files_fail(self, mock_abspath, mock_write_file,
|
||||
mock_process_permissions,
|
||||
mock_process_content):
|
||||
|
||||
self._cloudconfig.plugin_write_files([{}])
|
||||
self.assertEqual(0, mock_abspath.call_count)
|
||||
self.assertEqual(0, mock_process_permissions.call_count)
|
||||
self.assertEqual(0, mock_process_content.call_count)
|
||||
self.assertEqual(0, mock_write_file.call_count)
|
||||
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.CloudConfigPlugin.plugin_write_files')
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.CloudConfigPlugin._content')
|
||||
def test_process(self, mock_content, mock_plugin_write_files):
|
||||
mock_content.side_effect = [
|
||||
[("write_files", mock.sentinel.content)],
|
||||
[("invalid_plugin", mock.sentinel.content)]
|
||||
]
|
||||
mock_part = mock.sentinel.part
|
||||
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.base'
|
||||
'.BaseUserDataPlugin.get_mime_type')
|
||||
def test_process(self, mock_get_mime_type):
|
||||
mock_part = mock.MagicMock()
|
||||
self._cloudconfig.process(mock_part)
|
||||
mock_get_mime_type.assert_called_once_with()
|
||||
mock_plugin_write_files.assert_called_once_with(mock.sentinel.content)
|
||||
mock_content.assert_called_once_with(mock_part)
|
||||
|
||||
mock_plugin_write_files.reset_mock()
|
||||
self._cloudconfig.process(mock_part)
|
||||
self.assertEqual(0, mock_plugin_write_files.call_count)
|
||||
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.CloudConfigPlugin.plugin_write_files')
|
||||
@mock.patch('cloudbaseinit.plugins.windows.userdataplugins.cloudconfig'
|
||||
'.CloudConfigPlugin._content')
|
||||
def test_process_fail(self, mock_content, mock_plugin_write_files):
|
||||
mock_content.return_value = [("write_files", mock.sentinel.content)]
|
||||
mock_plugin_write_files.side_effect = [ValueError()]
|
||||
mock_part = mock.sentinel.part
|
||||
|
||||
self._cloudconfig.process(mock_part)
|
||||
mock_content.assert_called_once_with(mock_part)
|
||||
mock_plugin_write_files.assert_called_once_with(mock.sentinel.content)
|
||||
|
||||
@ -8,3 +8,4 @@ six>=1.7.0
|
||||
Babel>=1.3
|
||||
oauth
|
||||
netifaces
|
||||
PyYAML
|
||||
Loading…
x
Reference in New Issue
Block a user