x/cloud-init
Code Issues Proposed changes

changes to behavior on specifying keys.

Browse Source 
The most likely end user operation (or at least a valid one) for base64
encoding would be to encode the user-data, but leave all other values
as plaintext.

In order to facilitate that, the user can simply add:
 b64-user-data=true
to indicate that user-data is base64 encoded.

Other changes here are to change the cloud-config and metadata keynames
that are used.
  base64_all = boolean(True)
  base64_keys = [list, of, keys]

Fixed up tests to accomodate.
This commit is contained in:
Scott Moser 2013-08-24 00:07:35 -04:00
parent 7e2f9544ad
commit 214dd2745f
4 changed files with 149 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
cloudinit/sources/DataSourceSmartOS.py
View File

@ -35,7 +35,6 @@ import os
import os.path
import serial
DS_NAME = 'SmartOS'
DEF_TTY_LOC = '/dev/ttyS1'
DEF_TTY_TIMEOUT = 60
LOG = logging.getLogger(__name__)
@ -51,6 +50,7 @@ SMARTOS_ATTRIB_MAP = {
}
# These are values which will never be base64 encoded.
# They come from the cloud platform, not user
SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info',
'iptables_disable']
@ -60,17 +60,13 @@ class DataSourceSmartOS(sources.DataSource):
sources.DataSource.__init__(self, sys_cfg, distro, paths)
self.seed_dir = os.path.join(paths.seed_dir, 'sdc')
self.is_smartdc = None
self.base_64_encoded = []
self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC)
self.seed_timeout = self.sys_cfg.get("serial_timeout",
DEF_TTY_TIMEOUT)
self.all_base64 = False
if 'decode_base64' in self.ds_cfg:
self.all_base64 = self.ds_cfg['decode_base64']
self.smartos_no_base64 = SMARTOS_NO_BASE64
if 'no_base64_decode' in self.ds_cfg:
self.smartos_no_base64 = self.ds_cfg['no_base64_decode']
self.seed = self.ds_cfg.get("serial_device", DEF_TTY_LOC)
self.seed_timeout = self.ds_cfg.get("serial_timeout", DEF_TTY_TIMEOUT)
self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode',
SMARTOS_NO_BASE64)
self.b64_keys = self.ds_cfg.get('base64_keys', [])
self.b64_all = self.ds_cfg.get('base64_all', False)
def __str__(self):
root = sources.DataSource.__str__(self)
@ -92,38 +88,22 @@ class DataSourceSmartOS(sources.DataSource):
system_uuid, system_type = dmi_info
if 'smartdc' not in system_type.lower():
LOG.debug("Host is not on SmartOS")
LOG.debug("Host is not on SmartOS. system_type=%s", system_type)
return False
self.is_smartdc = True
md['instance-id'] = system_uuid
self.base_64_encoded = query_data('base_64_enocded',
self.seed,
self.seed_timeout,
strip=True)
if self.base_64_encoded:
self.base_64_encoded = str(self.base_64_encoded).split(',')
else:
self.base_64_encoded = []
b64_keys = self.query('base64_keys', strip=True, b64=False)
if b64_keys is not None:
self.b64_keys = [k.strip() for k in str(b64_keys).split(',')]
if not self.all_base64:
self.all_base64 = util.is_true(query_data('meta_encoded_base64',
self.seed,
self.seed_timeout,
strip=True))
b64_all = self.query('base64_all', strip=True, b64=False)
if b64_all is not None:
self.b64_all = util.is_true(b64_all)
for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems():
smartos_noun, strip = attribute
b64encoded = False
if self.all_base64 and \
(smartos_noun not in self.smartos_no_base64 and \
ci_noun not in self.smartos_no_base64):
b64encoded = True
md[ci_noun] = query_data(smartos_noun, self.seed,
self.seed_timeout, strip=strip,
b64encoded=b64encoded)
md[ci_noun] = self.query(smartos_noun, strip=strip)
if not md['local-hostname']:
md['local-hostname'] = system_uuid
@ -141,20 +121,16 @@ class DataSourceSmartOS(sources.DataSource):
def get_instance_id(self):
return self.metadata['instance-id']
def not_b64_var(self, var):
"""Return true if value is read as b64."""
if var in self.smartos_no_base64 or \
not self.all_base64:
return True
return False
def query(self, noun, strip=False, default=None, b64=None):
if b64 is None:
if noun in self.smartos_no_base64:
b64 = False
elif self.b64_all or noun in self.b64_keys:
b64 = True
def is_b64_var(self, var):
"""Return true if value is read as b64."""
if self.all_base64 or (
var not in self.smartos_no_base64 and
var in self.base_64_encoded):
return True
return False
return query_data(noun=noun, strip=strip, seed_device=self.seed,
seed_timeout=self.seed_timeout, default=default,
b64=b64)
def get_serial(seed_device, seed_timeout):
@ -176,7 +152,8 @@ def get_serial(seed_device, seed_timeout):
return ser
def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
def query_data(noun, seed_device, seed_timeout, strip=False, default=None,
b64=None):
"""Makes a request to via the serial console via "GET <NOUN>"
In the response, the first line is the status, while subsequent lines
@ -200,7 +177,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
if 'SUCCESS' not in status:
ser.close()
return None
return default
while not eom_found:
m = ser.readline()
@ -211,18 +188,23 @@ def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False):
ser.close()
if b64 is None:
b64 = query_data('b64-%s' % noun, seed_device=seed_device,
seed_timeout=seed_timeout, b64=False,
default=False, strip=True)
b64 = util.is_true(b64)
resp = None
if not strip:
resp = "".join(response)
elif b64encoded:
if b64 or strip:
resp = "".join(response).rstrip()
else:
resp = "".join(response).rstrip()
resp = "".join(response)
if b64encoded:
if b64:
try:
return base64.b64decode(resp)
except TypeError:
LOG.warn("Failed base64 decoding key '%s'", noun)
return resp
return resp

10
doc/examples/cloud-config-datasources.txt
View File

@ -56,4 +56,12 @@ datasource:
# a server on the other end. By default, the second serial console is the
# device. SmartOS also uses a serial timeout of 60 seconds.
serial_device: /dev/ttyS1
serial timeout: 60
serial_timeout: 60
# a list of keys that will not be base64 decoded even if base64_all
no_base64_decode: ['root_authorized_keys', 'motd_sys_info',
'iptables_disable']
# a plaintext, comma delimited list of keys whose values are b64 encoded
base64_keys: []
# a boolean indicating that all keys not in 'no_base64_decode' are encoded
base64_all: False

16
doc/sources/smartos/README.rst
View File

@ -53,14 +53,20 @@ are provided by SmartOS:
* enable_motd_sys_info
* iptables_disable
This list can be changed through system config of variable 'no_base64_decode'.
This means that user-script and user-data as well as other values can be
base64 encoded. Since Cloud-init can only guess as to whether or not something
is truly base64 encoded, the following meta-data keys are hints as to whether
or not to base64 decode something:
* decode_base64: Except for excluded keys, attempt to base64 decode
* base64_all: Except for excluded keys, attempt to base64 decode
the values. If the value fails to decode properly, it will be
returned in its text
* base_64_encoded: A comma deliminated list of which values are base64
encoded.
* no_base64_decode: This is a configuration setting (i.e. /etc/cloud/cloud.cfg.d)
that sets which values should not be base64 decoded.
* base64_keys: A comma deliminated list of which keys are base64 encoded.
* b64-<key>:
for any key, if there exists an entry in the metadata for 'b64-<key>'
Then 'b64-<key>' is expected to be a plaintext boolean indicating whether
or not its value is encoded.
* no_base64_decode: This is a configuration setting
(i.e. /etc/cloud/cloud.cfg.d) that sets which values should not be
base64 decoded.

155
tests/unittests/test_datasource/test_smartos.py
View File

@ -29,20 +29,17 @@ from cloudinit.sources import DataSourceSmartOS
from mocker import MockerTestCase
import uuid
mock_returns = {
MOCK_RETURNS = {
'hostname': 'test-host',
'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname',
'disable_iptables_flag': None,
'enable_motd_sys_info': None,
'system_uuid': str(uuid.uuid4()),
'smartdc': 'smartdc',
'test-var1': 'some data',
'user-data': """
#!/bin/sh
/bin/true
""",
'user-data': '\n'.join(['#!/bin/sh', '/bin/true', '']),
}
DMI_DATA_RETURN = (str(uuid.uuid4()), 'smartdc')
class MockSerial(object):
"""Fake a serial terminal for testing the code that
@ -50,14 +47,13 @@ class MockSerial(object):
port = None
def __init__(self, b64encode=False):
def __init__(self, mockdata):
self.last = None
self.last = None
self.new = True
self.count = 0
self.mocked_out = []
self.b64encode = b64encode
self.b64excluded = DataSourceSmartOS.SMARTOS_NO_BASE64
self.mockdata = mockdata
def open(self):
return True
@ -75,12 +71,12 @@ class MockSerial(object):
def readline(self):
if self.new:
self.new = False
if self.last in mock_returns:
if self.last in self.mockdata:
return 'SUCCESS\n'
else:
return 'NOTFOUND %s\n' % self.last
if self.last in mock_returns:
if self.last in self.mockdata:
if not self.mocked_out:
self.mocked_out = [x for x in self._format_out()]
print self.mocked_out
@ -90,21 +86,16 @@ class MockSerial(object):
return self.mocked_out[self.count - 1]
def _format_out(self):
if self.last in mock_returns:
_mret = mock_returns[self.last]
if self.b64encode and \
self.last not in self.b64excluded:
yield base64.b64encode(_mret)
if self.last in self.mockdata:
_mret = self.mockdata[self.last]
try:
for l in _mret.splitlines():
yield "%s\n" % l.rstrip()
except:
yield "%s\n" % _mret.rstrip()
else:
try:
for l in _mret.splitlines():
yield "%s\n" % l.rstrip()
except:
yield "%s\n" % _mret.rstrip()
yield '\n'
yield '.'
yield '\n'
class TestSmartOSDataSource(MockerTestCase):
@ -126,26 +117,36 @@ class TestSmartOSDataSource(MockerTestCase):
ret = apply_patches(patches)
self.unapply += ret
def _get_ds(self, b64encode=False, sys_cfg=None):
def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None):
mod = DataSourceSmartOS
if mockdata is None:
mockdata = MOCK_RETURNS
if dmi_data is None:
dmi_data = DMI_DATA_RETURN
def _get_serial(*_):
return MockSerial(b64encode=b64encode)
return MockSerial(mockdata)
def _dmi_data():
return mock_returns['system_uuid'], 'smartdc'
return dmi_data
if not sys_cfg:
if sys_cfg is None:
sys_cfg = {}
data = {'sys_cfg': sys_cfg}
if ds_cfg is not None:
sys_cfg['datasource'] = sys_cfg.get('datasource', {})
sys_cfg['datasource']['SmartOS'] = ds_cfg
self.apply_patches([(mod, 'get_serial', _get_serial)])
self.apply_patches([(mod, 'dmi_data', _dmi_data)])
dsrc = mod.DataSourceSmartOS(
data.get('sys_cfg', {}), distro=None, paths=self.paths)
dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None,
paths=self.paths)
return dsrc
def test_seed(self):
# default seed should be /dev/ttyS1
dsrc = self._get_ds()
ret = dsrc.get_data()
self.assertTrue(ret)
@ -158,78 +159,106 @@ class TestSmartOSDataSource(MockerTestCase):
self.assertTrue(dsrc.is_smartdc)
def test_no_base64(self):
sys_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
dsrc = self._get_ds(sys_cfg=sys_cfg)
ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
dsrc = self._get_ds(ds_cfg=ds_cfg)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertTrue(dsrc.not_b64_var('test-var'))
def test_uuid(self):
dsrc = self._get_ds()
dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEquals(mock_returns['system_uuid'],
dsrc.metadata['instance-id'])
self.assertEquals(DMI_DATA_RETURN[0], dsrc.metadata['instance-id'])
def test_root_keys(self):
dsrc = self._get_ds()
dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEquals(mock_returns['root_authorized_keys'],
self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
dsrc.metadata['public-keys'])
def test_hostname_b64(self):
dsrc = self._get_ds(b64encode=True)
dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEquals(base64.b64encode(mock_returns['hostname']),
self.assertEquals(MOCK_RETURNS['hostname'],
dsrc.metadata['local-hostname'])
def test_hostname(self):
dsrc = self._get_ds()
dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEquals(mock_returns['hostname'],
self.assertEquals(MOCK_RETURNS['hostname'],
dsrc.metadata['local-hostname'])
def test_base64(self):
"""This tests to make sure that SmartOS system key/value pairs
are not interpetted as being base64 encoded, while making
sure that the others are when 'decode_base64' is set"""
dsrc = self._get_ds(sys_cfg={'decode_base64': True},
b64encode=True)
def test_base64_all(self):
# metadata provided base64_all of true
my_returns = MOCK_RETURNS.copy()
my_returns['base64_all'] = "true"
for k in ('hostname', 'user-data'):
my_returns[k] = base64.b64encode(my_returns[k])
dsrc = self._get_ds(mockdata=my_returns)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEquals(mock_returns['hostname'],
self.assertEquals(MOCK_RETURNS['hostname'],
dsrc.metadata['local-hostname'])
self.assertEquals("%s" % mock_returns['user-data'],
self.assertEquals(MOCK_RETURNS['user-data'],
dsrc.userdata_raw)
self.assertEquals(mock_returns['root_authorized_keys'],
self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
dsrc.metadata['public-keys'])
self.assertEquals(mock_returns['disable_iptables_flag'],
self.assertEquals(MOCK_RETURNS['disable_iptables_flag'],
dsrc.metadata['iptables_disable'])
self.assertEquals(mock_returns['enable_motd_sys_info'],
self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'],
dsrc.metadata['motd_sys_info'])
def test_userdata(self):
dsrc = self._get_ds()
def test_b64_userdata(self):
my_returns = MOCK_RETURNS.copy()
my_returns['b64-user-data'] = "true"
my_returns['b64-hostname'] = "true"
for k in ('hostname', 'user-data'):
my_returns[k] = base64.b64encode(my_returns[k])
dsrc = self._get_ds(mockdata=my_returns)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEquals("%s\n" % mock_returns['user-data'],
dsrc.userdata_raw)
self.assertEquals(MOCK_RETURNS['hostname'],
dsrc.metadata['local-hostname'])
self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
self.assertEquals(MOCK_RETURNS['root_authorized_keys'],
dsrc.metadata['public-keys'])
def test_b64_keys(self):
my_returns = MOCK_RETURNS.copy()
my_returns['base64_keys'] = 'hostname,ignored'
for k in ('hostname',):
my_returns[k] = base64.b64encode(my_returns[k])
dsrc = self._get_ds(mockdata=my_returns)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEquals(MOCK_RETURNS['hostname'],
dsrc.metadata['local-hostname'])
self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
def test_userdata(self):
dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw)
def test_disable_iptables_flag(self):
dsrc = self._get_ds()
dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEquals(mock_returns['disable_iptables_flag'],
self.assertEquals(MOCK_RETURNS['disable_iptables_flag'],
dsrc.metadata['iptables_disable'])
def test_motd_sys_info(self):
dsrc = self._get_ds()
dsrc = self._get_ds(mockdata=MOCK_RETURNS)
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEquals(mock_returns['enable_motd_sys_info'],
self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'],
dsrc.metadata['motd_sys_info'])