x/cloud-init
Code Issues Proposed changes

Don't forget the rest of the files!

Browse Source
This commit is contained in:
Joshua Harlow 2014-02-01 22:58:58 -08:00
parent 97fe498e45
commit a884d3ddc5
4 changed files with 719 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
cloudinit/sources/DataSourceOpenStack.py Normal file
View File

@ -0,0 +1,163 @@
# vi: ts=4 expandtab
#
# Copyright (C) 2014 Yahoo! Inc.
#
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import time
from cloudinit import log as logging
from cloudinit import sources
from cloudinit import url_helper
from cloudinit import util
from cloudinit.sources.helpers import openstack
LOG = logging.getLogger(__name__)
# Various defaults/constants...
DEF_MD_URL = "http://169.254.169.254"
DEFAULT_IID = "iid-dsopenstack"
DEFAULT_METADATA = {
"instance-id": DEFAULT_IID,
}
VALID_DSMODES = ("net", "disabled")
class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
def __init__(self, sys_cfg, distro, paths):
super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths)
self.dsmode = 'net'
self.metadata_address = None
self.ssl_details = util.fetch_ssl_details(self.paths)
self.version = None
self.files = {}
def __str__(self):
root = sources.DataSource.__str__(self)
mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version)
return mstr
def _get_url_settings(self):
# TODO(harlowja): this is shared with ec2 datasource, we should just
# move it to a shared location instead...
ds_cfg = self.ds_cfg
if not ds_cfg:
ds_cfg = {}
max_wait = 120
try:
max_wait = int(ds_cfg.get("max_wait", max_wait))
except Exception:
util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
timeout = 50
try:
timeout = max(0, int(ds_cfg.get("timeout", timeout)))
except Exception:
util.logexc(LOG, "Failed to get timeout, using %s", timeout)
return (max_wait, timeout)
def wait_for_metadata_service(self):
ds_cfg = self.ds_cfg
if not ds_cfg:
ds_cfg = {}
urls = ds_cfg.get("metadata_urls", [DEF_MD_URL])
filtered = [x for x in urls if util.is_resolvable_url(x)]
if set(filtered) != set(urls):
LOG.debug("Removed the following from metadata urls: %s",
list((set(urls) - set(filtered))))
if len(filtered):
urls = filtered
else:
LOG.warn("Empty metadata url list! using default list")
urls = [DEF_MD_URL]
md_urls = []
url2base = {}
for url in urls:
md_url = url_helper.combine_url(url, 'openstack',
openstack.OS_LATEST,
'meta_data.json')
md_urls.append(md_url)
url2base[md_url] = url
(max_wait, timeout) = self._get_url_settings()
if max_wait <= 0:
return False
start_time = time.time()
avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
timeout=timeout,
status_cb=LOG.warn)
if avail_url:
LOG.debug("Using metadata source: '%s'", url2base[avail_url])
else:
LOG.critical("Giving up on md from %s after %s seconds",
md_urls, int(time.time() - start_time))
self.metadata_address = url2base.get(avail_url)
return bool(avail_url)
def get_data(self):
try:
if not self.wait_for_metadata_service():
return False
except IOError:
return False
try:
results = util.log_time(LOG.debug,
'Crawl of openstack metadata service',
read_metadata_service,
args=[self.metadata_address],
kwargs={'ssl_details': self.ssl_details})
except openstack.NonReadable:
return False
except openstack.BrokenMetadata:
util.logexc(LOG, "Broken metadata address %s",
self.metadata_address)
return False
user_dsmode = results.get('dsmode', None)
if user_dsmode not in VALID_DSMODES + (None,):
LOG.warn("User specified invalid mode: %s" % user_dsmode)
user_dsmode = None
if user_dsmode == 'disabled':
return False
md = results.get('metadata', {})
md = util.mergemanydict([md, DEFAULT_METADATA])
self.metadata = md
self.ec2_metadata = results.get('ec2-metadata')
self.userdata_raw = results.get('userdata')
self.version = results['version']
self.files.update(results.get('files', {}))
self.vendordata_raw = results.get('vendordata')
return True
def read_metadata_service(base_url, version=None, ssl_details=None):
reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
return reader.read_v2(version=version)
# Used to match classes to dependencies
datasources = [
(DataSourceOpenStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
]
# Return a list of data sources that match this set of dependencies
def get_datasource_list(depends):
return sources.list_from_depends(depends, datasources)

14
cloudinit/sources/helpers/__init__.py Normal file
View File

@ -0,0 +1,14 @@
# vi: ts=4 expandtab
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

420
cloudinit/sources/helpers/openstack.py Normal file
View File

@ -0,0 +1,420 @@
# vi: ts=4 expandtab
#
# Copyright (C) 2012 Canonical Ltd.
# Copyright (C) 2012 Yahoo! Inc.
#
# Author: Scott Moser <scott.moser@canonical.com>
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import abc
import base64
import copy
import functools
import os
from cloudinit import ec2_utils
from cloudinit import log as logging
from cloudinit import sources
from cloudinit import url_helper
from cloudinit import util
# For reference: http://tinyurl.com/laora4c
LOG = logging.getLogger(__name__)
FILES_V1 = {
# Path <-> (metadata key name, translator function, default value)
'etc/network/interfaces': ('network_config', lambda x: x, ''),
'meta.js': ('meta_js', util.load_json, {}),
"root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''),
}
KEY_COPIES = (
# Cloud-init metadata names <-> (metadata key, is required)
('local-hostname', 'hostname', False),
('instance-id', 'uuid', True),
)
OS_VERSIONS = (
'2012-08-10', # folsom
'2013-04-04', # grizzly
'2013-10-17', # havana
)
OS_LATEST = 'latest'
class NonReadable(IOError):
pass
class BrokenMetadata(IOError):
pass
class SourceMixin(object):
def _ec2_name_to_device(self, name):
if not self.ec2_metadata:
return None
bdm = self.ec2_metadata.get('block-device-mapping', {})
for (ent_name, device) in bdm.items():
if name == ent_name:
return device
return None
def get_public_ssh_keys(self):
name = "public_keys"
if self.version == 1:
name = "public-keys"
return sources.normalize_pubkey_data(self.metadata.get(name))
def _os_name_to_device(self, name):
device = None
try:
criteria = 'LABEL=%s' % (name)
if name == 'swap':
criteria = 'TYPE=%s' % (name)
dev_entries = util.find_devs_with(criteria)
if dev_entries:
device = dev_entries[0]
except util.ProcessExecutionError:
pass
return device
def _validate_device_name(self, device):
if not device:
return None
if not device.startswith("/"):
device = "/dev/%s" % device
if os.path.exists(device):
return device
# Durn, try adjusting the mapping
remapped = self._remap_device(os.path.basename(device))
if remapped:
LOG.debug("Remapped device name %s => %s", device, remapped)
return remapped
return None
def device_name_to_device(self, name):
# Translate a 'name' to a 'physical' device
if not name:
return None
# Try the ec2 mapping first
names = [name]
if name == 'root':
names.insert(0, 'ami')
if name == 'ami':
names.append('root')
device = None
LOG.debug("Using ec2 style lookup to find device %s", names)
for n in names:
device = self._ec2_name_to_device(n)
device = self._validate_device_name(device)
if device:
break
# Try the openstack way second
if not device:
LOG.debug("Using openstack style lookup to find device %s", names)
for n in names:
device = self._os_name_to_device(n)
device = self._validate_device_name(device)
if device:
break
# Ok give up...
if not device:
return None
else:
LOG.debug("Mapped %s to device %s", name, device)
return device
class BaseReader(object):
__metaclass__ = abc.ABCMeta
def __init__(self, base_path):
self.base_path = base_path
@abc.abstractmethod
def _path_join(self, base, *add_ons):
pass
@abc.abstractmethod
def _path_exists(self, path):
pass
@abc.abstractmethod
def _path_read(self, path):
pass
@abc.abstractmethod
def _read_ec2_metadata(self):
pass
def _read_content_path(self, item):
path = item.get('content_path', '').lstrip("/")
path_pieces = path.split("/")
valid_pieces = [p for p in path_pieces if len(p)]
if not valid_pieces:
raise BrokenMetadata("Item %s has no valid content path" % (item))
path = self._path_join(self.base_path, "openstack", *path_pieces)
return self._path_read(path)
def _find_working_version(self, version):
search_versions = [version] + list(OS_VERSIONS)
for potential_version in search_versions:
if not potential_version:
continue
path = self._path_join(self.base_path, "openstack",
potential_version)
if self._path_exists(path):
if potential_version != version:
LOG.warn("Version '%s' not available, attempting to use"
" version '%s' instead", version,
potential_version)
return potential_version
LOG.warn("Version '%s' not available, attempting to use '%s'"
" instead", version, OS_LATEST)
return OS_LATEST
def read_v2(self, version=None):
"""Reads a version 2 formatted location.
Return a dict with metadata, userdata, ec2-metadata, dsmode,
network_config, files and version (2).
If not a valid location, raise a NonReadable exception.
"""
def datafiles(version):
files = {}
files['metadata'] = (
# File path to read
self._path_join("openstack", version, 'meta_data.json'),
# Is it required?
True,
# Translator function (applied after loading)
util.load_json,
)
files['userdata'] = (
self._path_join("openstack", version, 'user_data'),
False,
lambda x: x,
)
files['vendordata'] = (
self._path_join("openstack", version, 'vendor_data.json'),
False,
util.load_json,
)
return files
version = self._find_working_version(version)
results = {
'userdata': '',
'version': 2,
}
data = datafiles(version)
for (name, (path, required, translator)) in data.iteritems():
path = self._path_join(self.base_path, path)
data = None
found = False
if self._path_exists(path):
try:
data = self._path_read(path)
except IOError:
raise NonReadable("Failed to read: %s" % path)
found = True
else:
if required:
raise NonReadable("Missing mandatory path: %s" % path)
if found and translator:
try:
data = translator(data)
except Exception as e:
raise BrokenMetadata("Failed to process "
"path %s: %s" % (path, e))
if found:
results[name] = data
metadata = results['metadata']
if 'random_seed' in metadata:
random_seed = metadata['random_seed']
try:
metadata['random_seed'] = base64.b64decode(random_seed)
except (ValueError, TypeError) as e:
raise BrokenMetadata("Badly formatted metadata"
" random_seed entry: %s" % e)
# load any files that were provided
files = {}
metadata_files = metadata.get('files', [])
for item in metadata_files:
if 'path' not in item:
continue
path = item['path']
try:
files[path] = self._read_content_path(item)
except Exception as e:
raise BrokenMetadata("Failed to read provided "
"file %s: %s" % (path, e))
results['files'] = files
# The 'network_config' item in metadata is a content pointer
# to the network config that should be applied. It is just a
# ubuntu/debian '/etc/network/interfaces' file.
net_item = metadata.get("network_config", None)
if net_item:
try:
results['network_config'] = self._read_content_path(net_item)
except IOError as e:
raise BrokenMetadata("Failed to read network"
" configuration: %s" % (e))
# To openstack, user can specify meta ('nova boot --meta=key=value')
# and those will appear under metadata['meta'].
# if they specify 'dsmode' they're indicating the mode that they intend
# for this datasource to operate in.
try:
results['dsmode'] = metadata['meta']['dsmode']
except KeyError:
pass
# Read any ec2-metadata (if applicable)
results['ec2-metadata'] = self._read_ec2_metadata()
# Perform some misc. metadata key renames...
for (target_key, source_key, is_required) in KEY_COPIES:
if is_required and source_key not in metadata:
raise BrokenMetadata("No '%s' entry in metadata" % source_key)
if source_key in metadata:
metadata[target_key] = metadata.get(source_key)
return results
class ConfigDriveReader(BaseReader):
def __init__(self, base_path):
super(ConfigDriveReader, self).__init__(base_path)
def _path_join(self, base, *add_ons):
components = [base] + list(add_ons)
return os.path.join(*components)
def _path_exists(self, path):
return os.path.exists(path)
def _path_read(self, path):
return util.load_file(path)
def _read_ec2_metadata(self):
path = self._path_join(self.base_path,
'ec2', 'latest', 'meta-data.json')
if not self._path_exists(path):
return {}
else:
try:
return util.load_json(self._path_read(path))
except Exception as e:
raise BrokenMetadata("Failed to process "
"path %s: %s" % (path, e))
def read_v1(self):
"""Reads a version 1 formatted location.
Return a dict with metadata, userdata, dsmode, files and version (1).
If not a valid path, raise a NonReadable exception.
"""
found = {}
for name in FILES_V1.keys():
path = self._path_join(self.base_path, name)
if self._path_exists(path):
found[name] = path
if len(found) == 0:
raise NonReadable("%s: no files found" % (self.base_path))
md = {}
for (name, (key, translator, default)) in FILES_V1.iteritems():
if name in found:
path = found[name]
try:
contents = self._path_read(path)
except IOError:
raise BrokenMetadata("Failed to read: %s" % path)
try:
md[key] = translator(contents)
except Exception as e:
raise BrokenMetadata("Failed to process "
"path %s: %s" % (path, e))
else:
md[key] = copy.deepcopy(default)
keydata = md['authorized_keys']
meta_js = md['meta_js']
# keydata in meta_js is preferred over "injected"
keydata = meta_js.get('public-keys', keydata)
if keydata:
lines = keydata.splitlines()
md['public-keys'] = [l for l in lines
if len(l) and not l.startswith("#")]
# config-drive-v1 has no way for openstack to provide the instance-id
# so we copy that into metadata from the user input
if 'instance-id' in meta_js:
md['instance-id'] = meta_js['instance-id']
results = {
'version': 1,
'metadata': md,
}
# allow the user to specify 'dsmode' in a meta tag
if 'dsmode' in meta_js:
results['dsmode'] = meta_js['dsmode']
# config-drive-v1 has no way of specifying user-data, so the user has
# to cheat and stuff it in a meta tag also.
results['userdata'] = meta_js.get('user-data', '')
# this implementation does not support files other than
# network/interfaces and authorized_keys...
results['files'] = {}
return results
class MetadataReader(BaseReader):
def __init__(self, base_url, ssl_details=None, timeout=5, retries=5):
super(MetadataReader, self).__init__(base_url)
self._url_reader = functools.partial(url_helper.readurl,
retries=retries,
ssl_details=ssl_details,
timeout=timeout)
self._url_checker = functools.partial(url_helper.existsurl,
ssl_details=ssl_details,
timeout=timeout)
self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata,
ssl_details=ssl_details,
timeout=timeout,
retries=retries)
def _path_read(self, path):
return str(self._url_reader(path))
def _path_exists(self, path):
return self._url_checker(path)
def _path_join(self, base, *add_ons):
return url_helper.combine_url(base, *add_ons)
def _read_ec2_metadata(self):
return self._ec2_reader()

122
tests/unittests/test_datasource/test_openstack.py Normal file
View File

@ -0,0 +1,122 @@
import re
import json
from StringIO import StringIO
from urlparse import urlparse
from tests.unittests import helpers
from cloudinit.sources import DataSourceOpenStack as ds
from cloudinit.sources.helpers import openstack
from cloudinit import util
import httpretty as hp
BASE_URL = "http://169.254.169.254"
PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n'
EC2_META = {
'ami-id': 'ami-00000001',
'ami-launch-index': 0,
'ami-manifest-path': 'FIXME',
'hostname': 'sm-foo-test.novalocal',
'instance-action': 'none',
'instance-id': 'i-00000001',
'instance-type': 'm1.tiny',
'local-hostname': 'sm-foo-test.novalocal',
'local-ipv4': '0.0.0.0',
'public-hostname': 'sm-foo-test.novalocal',
'public-ipv4': '0.0.0.1',
'reservation-id': 'r-iru5qm4m',
}
USER_DATA = '#!/bin/sh\necho This is user data\n'
VENDOR_DATA = {
'magic': '',
}
OSTACK_META = {
'availability_zone': 'nova',
'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'},
{'content_path': '/content/0001', 'path': '/etc/bar/bar.cfg'}],
'hostname': 'sm-foo-test.novalocal',
'meta': {'dsmode': 'local', 'my-meta': 'my-value'},
'name': 'sm-foo-test',
'public_keys': {'mykey': PUBKEY},
'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c',
}
CONTENT_0 = 'This is contents of /etc/foo.cfg\n'
CONTENT_1 = '# this is /etc/bar/bar.cfg\n'
OS_FILES = {
'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META),
'openstack/2012-08-10/user_data': USER_DATA,
'openstack/content/0000': CONTENT_0,
'openstack/content/0001': CONTENT_1,
'openstack/latest/meta_data.json': json.dumps(OSTACK_META),
'openstack/latest/user_data': USER_DATA,
'openstack/latest/vendor_data.json': json.dumps(VENDOR_DATA),
}
EC2_FILES = {
'latest/user-data': USER_DATA,
}
def _register_uris(version):
def match_ec2_url(uri, headers):
path = uri.path.lstrip("/")
if path in EC2_FILES:
return (200, headers, EC2_FILES.get(path))
if path == 'latest/meta-data':
buf = StringIO()
for (k, v) in EC2_META.items():
if isinstance(v, (list, tuple)):
buf.write("%s/" % (k))
else:
buf.write("%s" % (k))
buf.write("\n")
return (200, headers, buf.getvalue())
if path.startswith('latest/meta-data'):
value = None
pieces = path.split("/")
if path.endswith("/"):
pieces = pieces[2:-1]
value = util.get_cfg_by_path(EC2_META, pieces)
else:
pieces = pieces[2:]
value = util.get_cfg_by_path(EC2_META, pieces)
if value is not None:
return (200, headers, str(value))
return (404, headers, '')
def get_request_callback(method, uri, headers):
uri = urlparse(uri)
path = uri.path.lstrip("/")
if path in OS_FILES:
return (200, headers, OS_FILES.get(path))
return match_ec2_url(uri, headers)
def head_request_callback(method, uri, headers):
uri = urlparse(uri)
path = uri.path.lstrip("/")
for key in OS_FILES.keys():
if key.startswith(path):
return (200, headers, '')
return (404, headers, '')
hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'),
body=get_request_callback)
hp.register_uri(hp.HEAD, re.compile(r'http://169.254.169.254/.*'),
body=head_request_callback)
class TestOpenStackDataSource(helpers.TestCase):
VERSION = 'latest'
@hp.activate
def test_fetch(self):
_register_uris(self.VERSION)
f = ds.read_metadata_service(BASE_URL, version=self.VERSION)
self.assertEquals(VENDOR_DATA, f.get('vendordata'))
self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg'])
self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
self.assertEquals(USER_DATA, f.get('userdata'))