From c9ca5a35bc335ed48a2b64093c82c25774869dde Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 18 Oct 2012 10:34:38 -0700 Subject: [PATCH 01/22] Start fixing how boto and our url fetching can not use ssl correctly (they can not do validation due to the underlying usage of urllib/httplib). Adjust to use urllib3 which can in cloud-init url fetching. For now also take the metadata/userdata extraction code from boto (small amount) and fix it in a new local file called 'ec2_utils'. --- Requires | 3 + cloudinit/ec2_utils.py | 135 ++++++++++++++++++++++++++++++++++++++++ cloudinit/url_helper.py | 97 ++++++++++------------------- 3 files changed, 171 insertions(+), 64 deletions(-) create mode 100644 cloudinit/ec2_utils.py diff --git a/Requires b/Requires index 4f9311d5..b23dd4e9 100644 --- a/Requires +++ b/Requires @@ -26,3 +26,6 @@ pyyaml # The new main entrypoint uses argparse instead of optparse argparse + +# Urllib3 handles ssl correctly! +urllib3 diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py new file mode 100644 index 00000000..ef7fac7d --- /dev/null +++ b/cloudinit/ec2_utils.py @@ -0,0 +1,135 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import urllib + +from cloudinit import log as logging +from cloudinit import url_helper as uh +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +# For now take this and fix it... +class LazyLoadMetadata(dict): + def __init__(self, url, fetch_timeout, num_retries, ssl_details): + self._url = url + self._num_retries = num_retries + self._ssl_details = ssl_details + self._fetch_timeout = fetch_timeout + self._leaves = {} + self._dicts = [] + response = uh.readurl(url, timeout=fetch_timeout, + retries=num_retries, ssl_details=ssl_details) + data = str(response) + if data: + fields = data.split('\n') + for field in fields: + if field.endswith('/'): + key = field[0:-1] + self._dicts.append(key) + else: + p = field.find('=') + if p > 0: + key = field[p + 1:] + resource = field[0:p] + '/openssh-key' + else: + key = resource = field + self._leaves[key] = resource + self[key] = None + + def _materialize(self): + for key in self: + self[key] + + def __getitem__(self, key): + if key not in self: + # Allow dict to throw the KeyError + return super(LazyLoadMetadata, self).__getitem__(key) + + # Already loaded + val = super(LazyLoadMetadata, self).__getitem__(key) + if val is not None: + return val + + if key in self._leaves: + resource = self._leaves[key] + new_url = self._url + urllib.quote(resource, safe="/:") + response = uh.readurl(new_url, retries=self._num_retries, + timeout=self._fetch_timeout, + ssl_details=self._ssl_details) + val = str(response) + if val and val[0] == '{': + val = json.loads(val) + else: + p = val.find('\n') + if p > 0: + val = val.split('\n') + self[key] = val + elif key in self._dicts: + new_url = self._url + key + '/' + self[key] = LazyLoadMetadata(new_url, + num_retries=self._num_retries, + fetch_timeout=self._fetch_timeout, + ssl_details=self._ssl_details) + + return super(LazyLoadMetadata, self).__getitem__(key) + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def values(self): + self._materialize() + return super(LazyLoadMetadata, self).values() + + def items(self): + self._materialize() + return super(LazyLoadMetadata, self).items() + + def __str__(self): + self._materialize() + return super(LazyLoadMetadata, self).__str__() + + def __repr__(self): + self._materialize() + return super(LazyLoadMetadata, self).__repr__() + + +def get_instance_userdata(url, version='latest', ssl_details=None): + ud_url = '%s/%s/user-data' % (url, version) + try: + response = uh.readurl(ud_url, timeout=5, + retries=10, ssl_details=ssl_details) + return str(response) + except Exception as e: + util.logexc(LOG, "Failed fetching url %s", ud_url) + return None + + +def get_instance_metadata(url, version='latest', ssl_details=None): + md_url = '%s/%s/meta-data' % (url, version) + try: + return LazyLoadMetadata(md_url, timeout=5, + retries=10, ssl_details=ssl_details) + except Exception as e: + util.logexc(LOG, "Failed fetching url %s", md_url) + return None diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index f3e3fd7e..e3f63021 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -26,7 +26,9 @@ import errno import socket import time import urllib -import urllib2 + +from urllib3 import connectionpool +from urllib3 import util from cloudinit import log as logging from cloudinit import version @@ -68,71 +70,38 @@ class UrlResponse(object): return False -def readurl(url, data=None, timeout=None, - retries=0, sec_between=1, headers=None): - +def readurl(url, data=None, timeout=None, retries=0, + headers=None, ssl_details=None): req_args = {} - req_args['url'] = url - if data is not None: - req_args['data'] = urllib.urlencode(data) - - if not headers: - headers = { - 'User-Agent': 'Cloud-Init/%s' % (version.version_string()), + p_url = util.parse_url(url) + if p_url.scheme == 'https' and ssl_details: + for k in ['key_file', 'cert_file', 'cert_reqs', 'ca_certs']: + if k in ssl_details: + req_args[k] = ssl_details[k] + with closing(connectionpool.connection_from_url(url, **req_args)) as req_p: + retries = max(int(retries), 0) + attempts = retries + 1 + LOG.debug(("Attempting to open '%s' with %s attempts" + " (%s retries, timeout=%s) to be performed"), + url, attempts, retries, timeout) + open_args = { + 'method': 'GET', + 'retries': retries, + 'redirect': False, + 'url': p_url.request_uri, } - - req_args['headers'] = headers - req = urllib2.Request(**req_args) - - retries = max(retries, 0) - attempts = retries + 1 - - excepts = [] - LOG.debug(("Attempting to open '%s' with %s attempts" - " (%s retries, timeout=%s) to be performed"), - url, attempts, retries, timeout) - open_args = {} - if timeout is not None: - open_args['timeout'] = int(timeout) - for i in range(0, attempts): - try: - with closing(urllib2.urlopen(req, **open_args)) as rh: - content = rh.read() - status = rh.getcode() - if status is None: - # This seems to happen when files are read... - status = 200 - headers = {} - if rh.headers: - headers = dict(rh.headers) - LOG.debug("Read from %s (%s, %sb) after %s attempts", - url, status, len(content), (i + 1)) - return UrlResponse(status, content, headers) - except urllib2.HTTPError as e: - excepts.append(e) - except urllib2.URLError as e: - # This can be a message string or - # another exception instance - # (socket.error for remote URLs, OSError for local URLs). - if (isinstance(e.reason, (OSError)) and - e.reason.errno == errno.ENOENT): - excepts.append(e.reason) - else: - excepts.append(e) - except Exception as e: - excepts.append(e) - if i + 1 < attempts: - LOG.debug("Please wait %s seconds while we wait to try again", - sec_between) - time.sleep(sec_between) - - # Didn't work out - LOG.debug("Failed reading from %s after %s attempts", url, attempts) - - # It must of errored at least once for code - # to get here so re-raise the last error - LOG.debug("%s errors occured, re-raising the last one", len(excepts)) - raise excepts[-1] + if data is not None: + open_args['body'] = urllib.urlencode(data) + open_args['method'] = 'POST' + if not headers: + headers = { + 'User-Agent': 'Cloud-Init/%s' % (version.version_string()), + } + open_args['headers'] = headers + if timeout is not None: + open_args['timeout'] = max(int(timeout), 0) + r = req_p.urlopen(**open_args) + return UrlResponse(r.status, r.data, r.headers) def wait_for_url(urls, max_wait=None, timeout=None, From 5a17518016f36cb5040d3274640d446757235667 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 19 Oct 2012 14:06:21 -0700 Subject: [PATCH 02/22] Remove the need for boto just for fetching the userdata and metadata. Add in this crawling functionality to the ec2_utils module that will fully crawl (not lazily) the ec2 metadata and parse it in the same manner as boto. 1. Make the ec2 datasource + cloudstack now call into these. 2. Fix phone_home due to urllib3 change (TBD) --- Requires | 5 - cloudinit/config/cc_phone_home.py | 2 +- cloudinit/ec2_utils.py | 190 ++++++++++++---------- cloudinit/sources/DataSourceCloudStack.py | 9 +- cloudinit/sources/DataSourceEc2.py | 9 +- cloudinit/url_helper.py | 16 +- 6 files changed, 121 insertions(+), 110 deletions(-) diff --git a/Requires b/Requires index b23dd4e9..13a5d997 100644 --- a/Requires +++ b/Requires @@ -10,11 +10,6 @@ PrettyTable # datasource is removed, this is no longer needed oauth -# This is used to fetch the ec2 metadata into a easily -# parseable format, instead of having to have cloud-init perform -# those same fetchs and decodes and signing (...) that ec2 requires. -boto - # This is only needed for places where we need to support configs in a manner # that the built-in config parser is not sufficent (ie # when we need to preserve comments, or do not have a top-level diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index ae1349eb..a268d66f 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -112,7 +112,7 @@ def handle(name, cfg, cloud, log, args): } url = templater.render_string(url, url_params) try: - uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3) + uhelp.readurl(url, data=real_submit_keys, retries=tries) except: util.logexc(log, ("Failed to post phone home data to" " %s in %s tries"), url, tries) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index ef7fac7d..b9d7a2f7 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,6 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from urlparse import (urlparse, urlunparse) + import json import urllib @@ -26,110 +28,132 @@ from cloudinit import util LOG = logging.getLogger(__name__) -# For now take this and fix it... -class LazyLoadMetadata(dict): - def __init__(self, url, fetch_timeout, num_retries, ssl_details): - self._url = url - self._num_retries = num_retries - self._ssl_details = ssl_details - self._fetch_timeout = fetch_timeout - self._leaves = {} - self._dicts = [] - response = uh.readurl(url, timeout=fetch_timeout, - retries=num_retries, ssl_details=ssl_details) - data = str(response) - if data: - fields = data.split('\n') - for field in fields: - if field.endswith('/'): - key = field[0:-1] - self._dicts.append(key) - else: - p = field.find('=') - if p > 0: - key = field[p + 1:] - resource = field[0:p] + '/openssh-key' - else: - key = resource = field - self._leaves[key] = resource - self[key] = None +def combine_url(base, add_on): + base_parsed = list(urlparse(base)) + path = base_parsed[2] + if path and not path.endswith("/"): + path += "/" + path += urllib.quote(str(add_on), safe="/:") + base_parsed[2] = path + return urlunparse(base_parsed) - def _materialize(self): - for key in self: - self[key] - def __getitem__(self, key): - if key not in self: - # Allow dict to throw the KeyError - return super(LazyLoadMetadata, self).__getitem__(key) +# See: http://bit.ly/TyoUQs +class MetadataMaterializer(object): + def __init__(self, blob, base_url, **fetch_settings): + self._blob = blob + self._md = None + self._base_url = base_url + self._fetch_settings = fetch_settings - # Already loaded - val = super(LazyLoadMetadata, self).__getitem__(key) - if val is not None: - return val + def _parse(self, blob): + leaves = {} + children = [] + if not blob: + return (leaves, children) - if key in self._leaves: - resource = self._leaves[key] - new_url = self._url + urllib.quote(resource, safe="/:") - response = uh.readurl(new_url, retries=self._num_retries, - timeout=self._fetch_timeout, - ssl_details=self._ssl_details) - val = str(response) - if val and val[0] == '{': - val = json.loads(val) + def has_children(item): + if item.endswith("/"): + return True else: - p = val.find('\n') - if p > 0: - val = val.split('\n') - self[key] = val - elif key in self._dicts: - new_url = self._url + key + '/' - self[key] = LazyLoadMetadata(new_url, - num_retries=self._num_retries, - fetch_timeout=self._fetch_timeout, - ssl_details=self._ssl_details) + return False - return super(LazyLoadMetadata, self).__getitem__(key) + def get_name(item): + if item.endswith("/"): + return item.rstrip("/") + return item - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default + for field in blob.splitlines(): + field = field.strip() + field_name = get_name(field) + if not field or not field_name: + continue + if has_children(field): + if field_name not in children: + children.append(field_name) + else: + contents = field.split("=", 1) + resource = field_name + if len(contents) > 1: + # What a PITA... + (ident, sub_contents) = contents + checked_ident = util.safe_int(ident) + if checked_ident is not None: + resource = "%s/openssh-key" % (checked_ident) + field_name = sub_contents + leaves[field_name] = resource + return (leaves, children) - def values(self): - self._materialize() - return super(LazyLoadMetadata, self).values() + def materialize(self): + if self._md is not None: + return self._md + self._md = self._materialize(self._blob, self._base_url) + return self._md - def items(self): - self._materialize() - return super(LazyLoadMetadata, self).items() + def _fetch_url(self, url, **opts): + response = uh.readurl(url, **opts) + return str(response) - def __str__(self): - self._materialize() - return super(LazyLoadMetadata, self).__str__() + def _decode_leaf_blob(self, blob): + if not blob: + return blob + stripped_blob = blob.strip() + if stripped_blob.startswith("{") and stripped_blob.endswith("}"): + # Assume and try with json + try: + return json.loads(blob) + except (ValueError, TypeError): + pass + if blob.find("\n") != -1: + return blob.splitlines() + return blob - def __repr__(self): - self._materialize() - return super(LazyLoadMetadata, self).__repr__() + def _materialize(self, blob, base_url): + (leaves, children) = self._parse(blob) + child_contents = {} + for c in children: + child_url = combine_url(base_url, c) + if not child_url.endswith("/"): + child_url += "/" + child_blob = self._fetch_url(child_url, **self._fetch_settings) + child_contents[c] = self._materialize(child_blob, child_url) + leaf_contents = {} + for (field, resource) in leaves.items(): + leaf_url = combine_url(base_url, resource) + leaf_blob = self._fetch_url(leaf_url, **self._fetch_settings) + leaf_contents[field] = self._decode_leaf_blob(leaf_blob) + joined = {} + joined.update(child_contents) + for field in leaf_contents.keys(): + if field in joined: + LOG.warn("Duplicate key found in results from %s", base_url) + else: + joined[field] = leaf_contents[field] + return joined def get_instance_userdata(url, version='latest', ssl_details=None): - ud_url = '%s/%s/user-data' % (url, version) + ud_url = combine_url(url, version) + ud_url = combine_url(ud_url, 'user-data') try: response = uh.readurl(ud_url, timeout=5, retries=10, ssl_details=ssl_details) return str(response) - except Exception as e: - util.logexc(LOG, "Failed fetching url %s", ud_url) + except Exception: + util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return None def get_instance_metadata(url, version='latest', ssl_details=None): - md_url = '%s/%s/meta-data' % (url, version) + md_url = combine_url(url, version) + md_url = combine_url(md_url, 'meta-data') try: - return LazyLoadMetadata(md_url, timeout=5, - retries=10, ssl_details=ssl_details) - except Exception as e: - util.logexc(LOG, "Failed fetching url %s", md_url) + response = uh.readurl(md_url, timeout=5, + retries=10, ssl_details=ssl_details) + materializer = MetadataMaterializer(str(response), md_url, + timeout=5, retries=10, + ssl_details=ssl_details) + return materializer.materialize() + except Exception: + util.logexc(LOG, "Failed fetching metadata from url %s", md_url) return None diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index f7ffa7cb..2654df53 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -26,8 +26,7 @@ from struct import pack import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -116,10 +115,8 @@ class DataSourceCloudStack(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver) + self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 3686fa10..204963e7 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -23,8 +23,7 @@ import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -65,10 +64,8 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver) + self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index e3f63021..2c9d5eef 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -22,11 +22,10 @@ from contextlib import closing -import errno -import socket import time import urllib +from urllib3 import exceptions from urllib3 import connectionpool from urllib3 import util @@ -91,7 +90,10 @@ def readurl(url, data=None, timeout=None, retries=0, 'url': p_url.request_uri, } if data is not None: - open_args['body'] = urllib.urlencode(data) + if isinstance(data, (str, basestring)): + open_args['body'] = data + else: + open_args['body'] = urllib.urlencode(data) open_args['method'] = 'POST' if not headers: headers = { @@ -112,7 +114,7 @@ def wait_for_url(urls, max_wait=None, timeout=None, max_wait: roughly the maximum time to wait before giving up The max time is *actually* len(urls)*timeout as each url will be tried once and given the timeout provided. - timeout: the timeout provided to urllib2.urlopen + timeout: the timeout provided to urlopen status_cb: call method with string message when a url is not available headers_cb: call method with single argument of url to get headers for request. @@ -174,12 +176,8 @@ def wait_for_url(urls, max_wait=None, timeout=None, e = ValueError(reason) else: return url - except urllib2.HTTPError as e: + except exceptions.HTTPError as e: reason = "http error [%s]" % e.code - except urllib2.URLError as e: - reason = "url error [%s]" % e.reason - except socket.timeout as e: - reason = "socket timeout [%s]" % e except Exception as e: reason = "unexpected error [%s]" % e From 0d7c6696c281f59300eb99c65c88f0f339965636 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 21 Oct 2012 19:37:00 -0700 Subject: [PATCH 03/22] Use the python requests lib instead of urllib3 since it provides the same support (and it uses that library). Urllib3 doesn't seem to exist as its own package (requests hides it) so solve that by using requests and handling its oddness ourself. --- cloudinit/url_helper.py | 118 ++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 42 deletions(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 2c9d5eef..c5921147 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -20,20 +20,33 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from contextlib import closing - import time -import urllib -from urllib3 import exceptions -from urllib3 import connectionpool -from urllib3 import util +import requests +from requests import exceptions + +from urlparse import urlparse from cloudinit import log as logging from cloudinit import version LOG = logging.getLogger(__name__) +# Check if requests has ssl support (added in requests >= 0.8.8) +SSL_ENABLED = False +CONFIG_ENABLED = False # This was added in 0.7 +try: + import pkg_resources + from distutils.version import LooseVersion + _REQ = pkg_resources.get_distribution('requests') + _REQ_VER = LooseVersion(_REQ.version) + if _REQ_VER >= LooseVersion('0.8.8'): + SSL_ENABLED = True + if _REQ_VER >= LooseVersion('0.7.0'): + CONFIG_ENABLED = True +except: + pass + class UrlResponse(object): def __init__(self, status_code, contents=None, headers=None): @@ -70,40 +83,60 @@ class UrlResponse(object): def readurl(url, data=None, timeout=None, retries=0, - headers=None, ssl_details=None): - req_args = {} - p_url = util.parse_url(url) - if p_url.scheme == 'https' and ssl_details: - for k in ['key_file', 'cert_file', 'cert_reqs', 'ca_certs']: - if k in ssl_details: - req_args[k] = ssl_details[k] - with closing(connectionpool.connection_from_url(url, **req_args)) as req_p: - retries = max(int(retries), 0) - attempts = retries + 1 - LOG.debug(("Attempting to open '%s' with %s attempts" - " (%s retries, timeout=%s) to be performed"), - url, attempts, retries, timeout) - open_args = { - 'method': 'GET', - 'retries': retries, - 'redirect': False, - 'url': p_url.request_uri, - } - if data is not None: - if isinstance(data, (str, basestring)): - open_args['body'] = data + headers=None, ssl_details=None, check_status=True): + req_args = { + 'url': url, + } + if urlparse(url).scheme == 'https' and ssl_details: + if not SSL_ENABLED: + LOG.warn("SSL is not enabled, cert. verification can not occur!") + else: + if 'ca_certs' in ssl_details and ssl_details['ca_certs']: + req_args['verify'] = ssl_details['ca_certs'] else: - open_args['body'] = urllib.urlencode(data) - open_args['method'] = 'POST' - if not headers: - headers = { - 'User-Agent': 'Cloud-Init/%s' % (version.version_string()), - } - open_args['headers'] = headers - if timeout is not None: - open_args['timeout'] = max(int(timeout), 0) - r = req_p.urlopen(**open_args) - return UrlResponse(r.status, r.data, r.headers) + req_args['verify'] = True + if 'cert_file' in ssl_details and 'key_file' in ssl_details: + req_args['cert'] = [ssl_details['cert_file'], + ssl_details['key_file']] + req_args['allow_redirects'] = False + req_args['method'] = 'GET' + if timeout is not None: + req_args['timeout'] = max(float(timeout), 0) + if data: + req_args['method'] = 'POST' + # It doesn't seem like config + # was added in older library versions, thus we + # need to manually do the retries if it wasn't + manual_tries = 1 + if CONFIG_ENABLED: + req_config = {} + req_config['store_cookies'] = False + if retries: + req_config['max_retries'] = max(int(retries), 0) + req_args['config'] = req_config + else: + if retries: + manual_tries = max(int(retries) + 1, 1) + if not headers: + headers = { + 'User-Agent': 'Cloud-Init/%s' % (version.version_string()), + } + req_args['headers'] = headers + LOG.debug("Attempting to open '%s' with %s configuration", url, req_args) + if data: + # Do this after the log (it might be large) + req_args['data'] = data + last_excp = [] + for _i in range(0, manual_tries): + try: + r = requests.request(**req_args) + except exceptions.RequestException as e: + last_excp = [e] + if last_excp: + raise last_excp[-1] + if check_status: + r.raise_for_status() + return UrlResponse(r.status_code, r.content, r.headers) def wait_for_url(urls, max_wait=None, timeout=None, @@ -167,7 +200,8 @@ def wait_for_url(urls, max_wait=None, timeout=None, else: headers = {} - resp = readurl(url, headers=headers, timeout=timeout) + resp = readurl(url, headers=headers, timeout=timeout, + check_status=False) if not resp.contents: reason = "empty response [%s]" % (resp.code) e = ValueError(reason) @@ -176,8 +210,8 @@ def wait_for_url(urls, max_wait=None, timeout=None, e = ValueError(reason) else: return url - except exceptions.HTTPError as e: - reason = "http error [%s]" % e.code + except exceptions.RequestException as e: + reason = "request error [%s]" % e except Exception as e: reason = "unexpected error [%s]" % e From c0e683a313024c7788dbd4c12430a49db4883641 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 21 Oct 2012 19:38:29 -0700 Subject: [PATCH 04/22] Fix the requires after using requests. --- Requires | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Requires b/Requires index 13a5d997..0313d569 100644 --- a/Requires +++ b/Requires @@ -22,5 +22,5 @@ pyyaml # The new main entrypoint uses argparse instead of optparse argparse -# Urllib3 handles ssl correctly! -urllib3 +# Requests handles ssl correctly! +requests From 566f5ba55e64e134c9e72a1f59172a79937c5df4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 21 Oct 2012 20:44:23 -0700 Subject: [PATCH 05/22] Fix the checking and capturing of exceptions. --- cloudinit/url_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index c5921147..93df2510 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -130,12 +130,12 @@ def readurl(url, data=None, timeout=None, retries=0, for _i in range(0, manual_tries): try: r = requests.request(**req_args) + if check_status: + r.raise_for_status() except exceptions.RequestException as e: last_excp = [e] if last_excp: raise last_excp[-1] - if check_status: - r.raise_for_status() return UrlResponse(r.status_code, r.content, r.headers) From 925f2b40f5f2593565db583912090b497399adc8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 22 Oct 2012 13:24:25 -0700 Subject: [PATCH 06/22] More cleanups for using the requests module. 1. Handle our own retries (so that we can sleep in between) 2. Cleanup the url so that partially invalid (no scheme) urls will continue working. 3. Allow redirects option now a param. --- cloudinit/config/cc_phone_home.py | 2 +- cloudinit/url_helper.py | 103 +++++++++++++++--------------- 2 files changed, 54 insertions(+), 51 deletions(-) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index a268d66f..ae1349eb 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -112,7 +112,7 @@ def handle(name, cfg, cloud, log, args): } url = templater.render_string(url, url_params) try: - uhelp.readurl(url, data=real_submit_keys, retries=tries) + uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3) except: util.logexc(log, ("Failed to post phone home data to" " %s in %s tries"), url, tries) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 93df2510..e8330e24 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -25,7 +25,7 @@ import time import requests from requests import exceptions -from urlparse import urlparse +from urlparse import (urlparse, urlunparse) from cloudinit import log as logging from cloudinit import version @@ -48,42 +48,20 @@ except: pass -class UrlResponse(object): - def __init__(self, status_code, contents=None, headers=None): - self._status_code = status_code - self._contents = contents - self._headers = headers - - @property - def code(self): - return self._status_code - - @property - def contents(self): - return self._contents - - @property - def headers(self): - return self._headers - - def __str__(self): - if not self.contents: - return '' - else: - return str(self.contents) - - def ok(self, redirects_ok=False): - upper = 300 - if redirects_ok: - upper = 400 - if self.code >= 200 and self.code < upper: - return True - else: - return False +def _cleanurl(url): + parsed_url = list(urlparse(url, scheme='http')) + if not parsed_url[1] and parsed_url[2]: + # Swap these since this seems to be a common + # occurrence when given urls like 'www.google.com' + parsed_url[1] = parsed_url[2] + parsed_url[2] = '' + return urlunparse(parsed_url) -def readurl(url, data=None, timeout=None, retries=0, - headers=None, ssl_details=None, check_status=True): +def readurl(url, data=None, timeout=None, retries=0, sec_between=1, + headers=None, ssl_details=None, check_status=True, + allow_redirects=False): + url = _cleanurl(url) req_args = { 'url': url, } @@ -98,7 +76,8 @@ def readurl(url, data=None, timeout=None, retries=0, if 'cert_file' in ssl_details and 'key_file' in ssl_details: req_args['cert'] = [ssl_details['cert_file'], ssl_details['key_file']] - req_args['allow_redirects'] = False + + req_args['allow_redirects'] = allow_redirects req_args['method'] = 'GET' if timeout is not None: req_args['timeout'] = max(float(timeout), 0) @@ -107,16 +86,19 @@ def readurl(url, data=None, timeout=None, retries=0, # It doesn't seem like config # was added in older library versions, thus we # need to manually do the retries if it wasn't - manual_tries = 1 if CONFIG_ENABLED: - req_config = {} - req_config['store_cookies'] = False - if retries: - req_config['max_retries'] = max(int(retries), 0) + req_config = { + 'store_cookies': False, + } + # Don't use the retry support built-in + # since it doesn't allow for 'sleep_times' + # in between tries.... + # if retries: + # req_config['max_retries'] = max(int(retries), 0) req_args['config'] = req_config - else: - if retries: - manual_tries = max(int(retries) + 1, 1) + manual_tries = 1 + if retries: + manual_tries = max(int(retries) + 1, 1) if not headers: headers = { 'User-Agent': 'Cloud-Init/%s' % (version.version_string()), @@ -126,17 +108,38 @@ def readurl(url, data=None, timeout=None, retries=0, if data: # Do this after the log (it might be large) req_args['data'] = data - last_excp = [] - for _i in range(0, manual_tries): + if sec_between is None: + sec_between = -1 + excps = [] + # Handle retrying ourselves since the built-in support + # doesn't handle sleeping between tries... + for i in range(0, manual_tries): try: r = requests.request(**req_args) if check_status: r.raise_for_status() + contents = r.content + status = r.status_code + headers = r.headers + LOG.debug("Read from %s (%s, %sb) after %s attempts", url, + status, len(contents), (i + 1)) + # Doesn't seem like we can make it use a different + # subclass for responses, so add our own backward-compat + # attrs + if not hasattr(r, 'code'): + setattr(r, 'code', status) + if not hasattr(r, 'contents'): + setattr(r, 'contents', contents) + return r except exceptions.RequestException as e: - last_excp = [e] - if last_excp: - raise last_excp[-1] - return UrlResponse(r.status_code, r.content, r.headers) + excps.append(e) + if i + 1 < manual_tries and sec_between > 0: + LOG.debug("Please wait %s seconds while we wait to try again", + sec_between) + time.sleep(sec_between) + if excps: + raise excps[-1] + return None # Should throw before this... def wait_for_url(urls, max_wait=None, timeout=None, From 2ad41a09a103e791025814347d341bb263c1b921 Mon Sep 17 00:00:00 2001 From: harlowja Date: Tue, 19 Feb 2013 22:51:49 -0800 Subject: [PATCH 07/22] Continue working on integrating requests. --- cloudinit/config/cc_phone_home.py | 4 +- cloudinit/ec2_utils.py | 24 ++++------ cloudinit/sources/DataSourceEc2.py | 2 +- cloudinit/sources/DataSourceMAAS.py | 16 ++++--- cloudinit/url_helper.py | 50 +++++++++++++++++--- cloudinit/user_data.py | 7 ++- cloudinit/util.py | 71 +++++++++++++++++++++++++---- 7 files changed, 129 insertions(+), 45 deletions(-) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index ae1349eb..90834080 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -19,7 +19,6 @@ # along with this program. If not, see . from cloudinit import templater -from cloudinit import url_helper as uhelp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -112,7 +111,8 @@ def handle(name, cfg, cloud, log, args): } url = templater.render_string(url, url_params) try: - uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3) + util.read_file_or_url(url, data=real_submit_keys, + retries=tries, sec_between=3) except: util.logexc(log, ("Failed to post phone home data to" " %s in %s tries"), url, tries) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index b9d7a2f7..c422eea9 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -22,7 +22,6 @@ import json import urllib from cloudinit import log as logging -from cloudinit import url_helper as uh from cloudinit import util LOG = logging.getLogger(__name__) @@ -40,11 +39,10 @@ def combine_url(base, add_on): # See: http://bit.ly/TyoUQs class MetadataMaterializer(object): - def __init__(self, blob, base_url, **fetch_settings): + def __init__(self, blob, base_url): self._blob = blob self._md = None self._base_url = base_url - self._fetch_settings = fetch_settings def _parse(self, blob): leaves = {} @@ -90,8 +88,8 @@ class MetadataMaterializer(object): self._md = self._materialize(self._blob, self._base_url) return self._md - def _fetch_url(self, url, **opts): - response = uh.readurl(url, **opts) + def _fetch_url(self, url): + response = util.read_file_or_url(url) return str(response) def _decode_leaf_blob(self, blob): @@ -115,12 +113,12 @@ class MetadataMaterializer(object): child_url = combine_url(base_url, c) if not child_url.endswith("/"): child_url += "/" - child_blob = self._fetch_url(child_url, **self._fetch_settings) + child_blob = self._fetch_url(child_url) child_contents[c] = self._materialize(child_blob, child_url) leaf_contents = {} for (field, resource) in leaves.items(): leaf_url = combine_url(base_url, resource) - leaf_blob = self._fetch_url(leaf_url, **self._fetch_settings) + leaf_blob = self._fetch_url(leaf_url) leaf_contents[field] = self._decode_leaf_blob(leaf_blob) joined = {} joined.update(child_contents) @@ -136,23 +134,19 @@ def get_instance_userdata(url, version='latest', ssl_details=None): ud_url = combine_url(url, version) ud_url = combine_url(ud_url, 'user-data') try: - response = uh.readurl(ud_url, timeout=5, - retries=10, ssl_details=ssl_details) + response = util.read_file_or_url(ud_url) return str(response) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return None -def get_instance_metadata(url, version='latest', ssl_details=None): +def get_instance_metadata(url, version='latest'): md_url = combine_url(url, version) md_url = combine_url(md_url, 'meta-data') try: - response = uh.readurl(md_url, timeout=5, - retries=10, ssl_details=ssl_details) - materializer = MetadataMaterializer(str(response), md_url, - timeout=5, retries=10, - ssl_details=ssl_details) + response = util.read_file_or_url(md_url) + materializer = MetadataMaterializer(str(response), md_url) return materializer.materialize() except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", md_url) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 204963e7..47f677d4 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -137,7 +137,7 @@ class DataSourceEc2(sources.DataSource): start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url2base[url]) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index e187aec9..2de31305 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -25,9 +25,11 @@ import os import time import urllib2 +import requests + from cloudinit import log as logging from cloudinit import sources -from cloudinit import url_helper as uhelp +from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) @@ -191,8 +193,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, version=MD_VERSION): """ Read the maas datasource at seed_url. - header_cb is a method that should return a headers dictionary that will - be given to urllib2.Request() + - header_cb is a method that should return a headers dictionary for + a given url Expected format of seed_url is are the following files: * //meta-data/instance-id @@ -220,13 +222,13 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, else: headers = {} try: - resp = uhelp.readurl(url, headers=headers, timeout=timeout) - if resp.ok(): + resp = util.read_file_or_url(url, headers=headers, timeout=timeout) + if resp.ok: md[name] = str(resp) else: LOG.warn(("Fetching from %s resulted in" - " an invalid http code %s"), url, resp.code) - except urllib2.HTTPError as e: + " an invalid http code %s"), url, resp.status_code) + except url_helper.UrlError as e: if e.code != 404: raise return check_seed_contents(md, seed_url) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index e8330e24..0839e63b 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -58,6 +58,44 @@ def _cleanurl(url): return urlunparse(parsed_url) +class UrlResponse(object): + def __init__(self, response): + self._response = response + + @property + def contents(self): + return self._response.content + + @property + def url(self): + return self._response.url + + @property + def ok(self): + return self._response.ok + + @property + def headers(self): + return self._response.headers + + @property + def code(self): + return self._response.status_code + + def __str__(self): + return self.contents + + +class UrlError(IOError): + def __init__(self, cause): + IOError.__init__(self, str(cause)) + self.cause = cause + if isinstance(cause, exceptions.HTTPError) and cause.response: + self.code = cause.response.status_code + else: + self.code = None + + def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, ssl_details=None, check_status=True, allow_redirects=False): @@ -76,6 +114,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, if 'cert_file' in ssl_details and 'key_file' in ssl_details: req_args['cert'] = [ssl_details['cert_file'], ssl_details['key_file']] + elif 'cert_file' in ssl_details: + req_args['cert'] = str(ssl_details['cert_file']) req_args['allow_redirects'] = allow_redirects req_args['method'] = 'GET' @@ -126,13 +166,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # Doesn't seem like we can make it use a different # subclass for responses, so add our own backward-compat # attrs - if not hasattr(r, 'code'): - setattr(r, 'code', status) - if not hasattr(r, 'contents'): - setattr(r, 'contents', contents) - return r + return UrlResponse(r) except exceptions.RequestException as e: - excps.append(e) + excps.append(UrlError(e)) if i + 1 < manual_tries and sec_between > 0: LOG.debug("Please wait %s seconds while we wait to try again", sec_between) @@ -213,7 +249,7 @@ def wait_for_url(urls, max_wait=None, timeout=None, e = ValueError(reason) else: return url - except exceptions.RequestException as e: + except UrlError as e: reason = "request error [%s]" % e except Exception as e: reason = "unexpected error [%s]" % e diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 803ffc3a..4a640f1e 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -29,7 +29,6 @@ from email.mime.text import MIMEText from cloudinit import handlers from cloudinit import log as logging -from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) @@ -173,10 +172,10 @@ class UserDataProcessor(object): if include_once_on and os.path.isfile(include_once_fn): content = util.load_file(include_once_fn) else: - resp = url_helper.readurl(include_url) - if include_once_on and resp.ok(): + resp = util.read_file_or_url(include_url) + if include_once_on and resp.ok: util.write_file(include_once_fn, str(resp), mode=0600) - if resp.ok(): + if resp.ok: content = str(resp) else: LOG.warn(("Fetching from %s resulted in" diff --git a/cloudinit/util.py b/cloudinit/util.py index f5a7ac12..da2cdeda 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -51,7 +51,7 @@ import yaml from cloudinit import importer from cloudinit import log as logging from cloudinit import safeyaml -from cloudinit import url_helper as uhelp +from cloudinit import url_helper from cloudinit.settings import (CFG_BUILTIN) @@ -69,6 +69,18 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters) CONTAINER_TESTS = ['running-in-container', 'lxc-is-container'] +class FileResponse(object): + def __init__(self, path, contents): + self.code = 200 + self.headers = {} + self.contents = contents + self.ok = True + self.url = path + + def __str__(self): + return self.contents + + class ProcessExecutionError(IOError): MESSAGE_TMPL = ('%(description)s\n' @@ -628,12 +640,53 @@ def read_optional_seed(fill, base="", ext="", timeout=5): raise -def read_file_or_url(url, timeout=5, retries=10, file_retries=0): +def fetch_ssl_details(paths=None): + ssl_details = {} + # Lookup in these locations for ssl key/cert files + ssl_cert_paths = [ + '/var/lib/cloud/data/ssl', + '/var/lib/cloud/instance/data/ssl', + ] + if paths: + ssl_cert_paths.extend([ + os.path.join(paths.get_ipath_cur('data'), 'ssl'), + os.path.join(paths.get_cpath('data'), 'ssl'), + ]) + ssl_cert_paths = uniq_merge(ssl_cert_paths) + ssl_cert_paths = [d for d in ssl_cert_paths if d and os.path.isdir(d)] + cert_file = None + for d in ssl_cert_paths: + if os.path.isfile(os.path.join(d, 'cert.pem')): + cert_file = os.path.join(d, 'cert.pem') + break + key_file = None + for d in ssl_cert_paths: + if os.path.isfile(os.path.join(d, 'key.pem')): + key_file = os.path.join(d, 'key.pem') + break + if cert_file and key_file: + ssl_details['cert_file'] = cert_file + ssl_details['key_file'] = key_file + elif cert_file: + ssl_details['cert_file'] = cert_file + return ssl_details + + +def read_file_or_url(url, timeout=5, retries=10, + headers=None, data=None, sec_between=1, paths=None): if url.startswith("/"): url = "file://%s" % url - if url.startswith("file://"): - retries = file_retries - return uhelp.readurl(url, timeout=timeout, retries=retries) + if url.lower().startswith("file://"): + file_path = url[len("file://"):] + return FileResponse(file_path, contents=load_file(file_path)) + else: + return url_helper.readurl(url, + timeout=timeout, + retries=retries, + headers=headers, + data=data, + sec_between=sec_between, + ssl_details=fetch_ssl_details(paths)) def load_yaml(blob, default=None, allowed=(dict,)): @@ -675,13 +728,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): md_resp = read_file_or_url(md_url, timeout, retries, file_retries) md = None - if md_resp.ok(): + if md_resp.ok: md_str = str(md_resp) md = load_yaml(md_str, default={}) ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries) ud = None - if ud_resp.ok(): + if ud_resp.ok: ud_str = str(ud_resp) ud = ud_str @@ -850,8 +903,8 @@ def get_cmdline_url(names=('cloud-config-url', 'url'), if not url: return (None, None, None) - resp = uhelp.readurl(url) - if resp.contents.startswith(starts) and resp.ok(): + resp = read_file_or_url(url) + if resp.contents.startswith(starts) and resp.ok: return (key, url, str(resp)) return (key, url, None) From caa625d3d7cd46de7925f61d8616ac9808bda799 Mon Sep 17 00:00:00 2001 From: harlowja Date: Thu, 21 Feb 2013 22:39:30 -0800 Subject: [PATCH 08/22] More work on requests integration. --- cloudinit/config/cc_phone_home.py | 3 ++- cloudinit/ec2_utils.py | 17 +++++++++++------ cloudinit/sources/DataSourceEc2.py | 6 ++++-- cloudinit/sources/DataSourceMAAS.py | 15 +++++++++------ cloudinit/util.py | 4 ++-- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 90834080..5a4332ef 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -112,7 +112,8 @@ def handle(name, cfg, cloud, log, args): url = templater.render_string(url, url_params) try: util.read_file_or_url(url, data=real_submit_keys, - retries=tries, sec_between=3) + retries=tries, sec_between=3, + ssl_details=util.fetch_ssl_details(cloud.paths)) except: util.logexc(log, ("Failed to post phone home data to" " %s in %s tries"), url, tries) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index c422eea9..026ee178 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -38,11 +38,16 @@ def combine_url(base, add_on): # See: http://bit.ly/TyoUQs +# +# Since boto metadata reader uses the old urllib which does not +# support ssl, we need to ahead and create our own reader which +# works the same as the boto one (for now). class MetadataMaterializer(object): - def __init__(self, blob, base_url): + def __init__(self, blob, base_url, ssl_details): self._blob = blob self._md = None self._base_url = base_url + self._ssl_details = ssl_details def _parse(self, blob): leaves = {} @@ -89,7 +94,7 @@ class MetadataMaterializer(object): return self._md def _fetch_url(self, url): - response = util.read_file_or_url(url) + response = util.read_file_or_url(url, ssl_details=self._ssl_details) return str(response) def _decode_leaf_blob(self, blob): @@ -134,19 +139,19 @@ def get_instance_userdata(url, version='latest', ssl_details=None): ud_url = combine_url(url, version) ud_url = combine_url(ud_url, 'user-data') try: - response = util.read_file_or_url(ud_url) + response = util.read_file_or_url(ud_url, ssl_details=ssl_details) return str(response) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return None -def get_instance_metadata(url, version='latest'): +def get_instance_metadata(url, version='latest', ssl_details=None): md_url = combine_url(url, version) md_url = combine_url(md_url, 'meta-data') try: - response = util.read_file_or_url(md_url) - materializer = MetadataMaterializer(str(response), md_url) + response = util.read_file_or_url(md_url, ssl_details=ssl_details) + materializer = MetadataMaterializer(str(response), md_url, ssl_details) return materializer.materialize() except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", md_url) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 47f677d4..eee4e6bc 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -64,8 +64,10 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver) - self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver) + self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver, + ssl_details=util.fetch_ssl_details(self.paths)) + self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver, + ssl_details=util.fetch_ssl_details(self.paths)) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 2de31305..dc048943 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -81,7 +81,8 @@ class DataSourceMAAS(sources.DataSource): self.base_url = url (userdata, metadata) = read_maas_seed_url(self.base_url, - self.md_headers) + self.md_headers, + paths=self.paths) self.userdata_raw = userdata self.metadata = metadata return True @@ -141,7 +142,7 @@ class DataSourceMAAS(sources.DataSource): LOG.debug("Using metadata source: '%s'", url) else: LOG.critical("Giving up on md from %s after %i seconds", - urls, int(time.time() - starttime)) + urls, int(time.time() - starttime)) return bool(url) @@ -190,7 +191,7 @@ def read_maas_seed_dir(seed_d): def read_maas_seed_url(seed_url, header_cb=None, timeout=None, - version=MD_VERSION): + version=MD_VERSION, paths=None): """ Read the maas datasource at seed_url. - header_cb is a method that should return a headers dictionary for @@ -222,12 +223,13 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, else: headers = {} try: - resp = util.read_file_or_url(url, headers=headers, timeout=timeout) + resp = util.read_file_or_url(url, headers=headers, timeout=timeout, + ssl_details=util.fetch_ssl_details(paths)) if resp.ok: md[name] = str(resp) else: LOG.warn(("Fetching from %s resulted in" - " an invalid http code %s"), url, resp.status_code) + " an invalid http code %s"), url, resp.code) except url_helper.UrlError as e: if e.code != 404: raise @@ -372,7 +374,8 @@ if __name__ == "__main__": if args.subcmd == "check-seed": if args.url.startswith("http"): (userdata, metadata) = read_maas_seed_url(args.url, - header_cb=my_headers, version=args.apiver) + header_cb=my_headers, + version=args.apiver) else: (userdata, metadata) = read_maas_seed_url(args.url) print "=== userdata ===" diff --git a/cloudinit/util.py b/cloudinit/util.py index da2cdeda..307ed613 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -673,7 +673,7 @@ def fetch_ssl_details(paths=None): def read_file_or_url(url, timeout=5, retries=10, - headers=None, data=None, sec_between=1, paths=None): + headers=None, data=None, sec_between=1, ssl_details=None): if url.startswith("/"): url = "file://%s" % url if url.lower().startswith("file://"): @@ -686,7 +686,7 @@ def read_file_or_url(url, timeout=5, retries=10, headers=headers, data=data, sec_between=sec_between, - ssl_details=fetch_ssl_details(paths)) + ssl_details=ssl_details) def load_yaml(blob, default=None, allowed=(dict,)): From f82bfbd3b3b4246140204c13daff832aacbd2c3e Mon Sep 17 00:00:00 2001 From: harlowja Date: Thu, 21 Feb 2013 22:56:17 -0800 Subject: [PATCH 09/22] Why did this file showup. --- cloudinit/ec2_utils.py.moved | 157 ----------------------------------- 1 file changed, 157 deletions(-) delete mode 100644 cloudinit/ec2_utils.py.moved diff --git a/cloudinit/ec2_utils.py.moved b/cloudinit/ec2_utils.py.moved deleted file mode 100644 index 4812eccb..00000000 --- a/cloudinit/ec2_utils.py.moved +++ /dev/null @@ -1,157 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from urlparse import (urlparse, urlunparse) - -import json -import urllib - -from cloudinit import log as logging -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -def combine_url(base, add_on): - base_parsed = list(urlparse(base)) - path = base_parsed[2] - if path and not path.endswith("/"): - path += "/" - path += urllib.quote(str(add_on), safe="/:") - base_parsed[2] = path - return urlunparse(base_parsed) - - -# See: http://bit.ly/TyoUQs -# -# Since boto metadata reader uses the old urllib which does not -# support ssl, we need to ahead and create our own reader which -# works the same as the boto one (for now). -class MetadataMaterializer(object): - def __init__(self, blob, base_url, ssl_details): - self._blob = blob - self._md = None - self._base_url = base_url - self._ssl_details = ssl_details - - def _parse(self, blob): - leaves = {} - children = [] - if not blob: - return (leaves, children) - - def has_children(item): - if item.endswith("/"): - return True - else: - return False - - def get_name(item): - if item.endswith("/"): - return item.rstrip("/") - return item - - for field in blob.splitlines(): - field = field.strip() - field_name = get_name(field) - if not field or not field_name: - continue - if has_children(field): - if field_name not in children: - children.append(field_name) - else: - contents = field.split("=", 1) - resource = field_name - if len(contents) > 1: - # What a PITA... - (ident, sub_contents) = contents - checked_ident = util.safe_int(ident) - if checked_ident is not None: - resource = "%s/openssh-key" % (checked_ident) - field_name = sub_contents - leaves[field_name] = resource - return (leaves, children) - - def materialize(self): - if self._md is not None: - return self._md - self._md = self._materialize(self._blob, self._base_url) - return self._md - - def _fetch_url(self, url): - response = util.read_file_or_url(url, ssl_details=self._ssl_details) - return str(response) - - def _decode_leaf_blob(self, blob): - if not blob: - return blob - stripped_blob = blob.strip() - if stripped_blob.startswith("{") and stripped_blob.endswith("}"): - # Assume and try with json - try: - return json.loads(blob) - except (ValueError, TypeError): - pass - if blob.find("\n") != -1: - return blob.splitlines() - return blob - - def _materialize(self, blob, base_url): - (leaves, children) = self._parse(blob) - child_contents = {} - for c in children: - child_url = combine_url(base_url, c) - if not child_url.endswith("/"): - child_url += "/" - child_blob = self._fetch_url(child_url) - child_contents[c] = self._materialize(child_blob, child_url) - leaf_contents = {} - for (field, resource) in leaves.items(): - leaf_url = combine_url(base_url, resource) - leaf_blob = self._fetch_url(leaf_url) - leaf_contents[field] = self._decode_leaf_blob(leaf_blob) - joined = {} - joined.update(child_contents) - for field in leaf_contents.keys(): - if field in joined: - LOG.warn("Duplicate key found in results from %s", base_url) - else: - joined[field] = leaf_contents[field] - return joined - - -def get_instance_userdata(api_version, metadata_address, ssl_details=None): - ud_url = combine_url(metadata_address, api_version) - ud_url = combine_url(ud_url, 'user-data') - try: - response = util.read_file_or_url(ud_url, ssl_details=ssl_details) - return str(response) - except Exception: - util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) - return None - -def get_instance_metadata(api_version, metadata_address, ssl_details=None): - md_url = combine_url(metadata_address, api_version) - md_url = combine_url(md_url, 'meta-data') - try: - response = util.read_file_or_url(md_url, ssl_details=ssl_details) - materializer = MetadataMaterializer(str(response), md_url, ssl_details) - return materializer.materialize() - except Exception: - util.logexc(LOG, "Failed fetching metadata from url %s", md_url) - return None From 5b9e1ce21d3fafbca40cfb99bf8fd5de7294fb7f Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 23 Feb 2013 21:23:24 -0800 Subject: [PATCH 10/22] Get tests working and further adjustments. --- cloudinit/sources/DataSourceMAAS.py | 9 ++++-- cloudinit/url_helper.py | 21 ++++++------ cloudinit/user_data.py | 8 +++-- cloudinit/util.py | 34 ++++++++++++++------ tests/unittests/test__init__.py | 10 +++--- tests/unittests/test_datasource/test_maas.py | 11 ++++--- 6 files changed, 60 insertions(+), 33 deletions(-) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 6e1133b2..0c526305 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -223,9 +223,12 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, else: headers = {} try: - resp = util.read_file_or_url(url, headers=headers, timeout=timeout, - ssl_details=util.fetch_ssl_details(paths)) - if resp.ok: + ssl_details = util.fetch_ssl_details(paths) + resp = util.read_file_or_url(url, + headers=headers, + timeout=timeout, + ssl_details=ssl_details) + if resp.ok(): md[name] = str(resp) else: LOG.warn(("Fetching from %s resulted in" diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 0839e63b..300e70c2 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -70,9 +70,14 @@ class UrlResponse(object): def url(self): return self._response.url - @property - def ok(self): - return self._response.ok + def ok(self, redirects_ok=False): + upper = 300 + if redirects_ok: + upper = 400 + if self.code >= 200 and self.code < upper: + return True + else: + return False @property def headers(self): @@ -158,11 +163,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, r = requests.request(**req_args) if check_status: r.raise_for_status() - contents = r.content - status = r.status_code - headers = r.headers LOG.debug("Read from %s (%s, %sb) after %s attempts", url, - status, len(contents), (i + 1)) + r.status_code, len(r.content), (i + 1)) # Doesn't seem like we can make it use a different # subclass for responses, so add our own backward-compat # attrs @@ -256,8 +258,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, time_taken = int(time.time() - start_time) status_msg = "Calling '%s' failed [%s/%ss]: %s" % (url, - time_taken, - max_wait, reason) + time_taken, + max_wait, + reason) status_cb(status_msg) if exception_cb: exception_cb(msg=status_msg, exception=e) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index eaf448a7..df069ff8 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -59,6 +59,7 @@ EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config"] class UserDataProcessor(object): def __init__(self, paths): self.paths = paths + self.ssl_details = util.fetch_ssl_details(paths) def process(self, blob): accumulating_msg = MIMEMultipart() @@ -172,10 +173,11 @@ class UserDataProcessor(object): if include_once_on and os.path.isfile(include_once_fn): content = util.load_file(include_once_fn) else: - resp = util.read_file_or_url(include_url) - if include_once_on and resp.ok: + resp = util.read_file_or_url(include_url, + ssl_details=self.ssl_details) + if include_once_on and resp.ok(): util.write_file(include_once_fn, str(resp), mode=0600) - if resp.ok: + if resp.ok(): content = str(resp) else: LOG.warn(("Fetching from %s resulted in" diff --git a/cloudinit/util.py b/cloudinit/util.py index 42b3ab01..dc3c5639 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -70,18 +70,31 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters) CONTAINER_TESTS = ['running-in-container', 'lxc-is-container'] -class FileResponse(object): - def __init__(self, path, contents): - self.code = 200 +# Made to have same accessors as UrlResponse so that the +# read_file_or_url can return this or that object and the +# 'user' of those objects will not need to know the difference. +class StringResponse(object): + def __init__(self, contents, code=200): + self.code = code self.headers = {} self.contents = contents - self.ok = True - self.url = path + self.url = None + + def ok(self, *args, **kwargs): + if self.code != 200: + return False + return True def __str__(self): return self.contents +class FileResponse(StringResponse): + def __init__(self, path, contents, code=200): + StringResponse.__init__(self, contents, code=code) + self.url = path + + class ProcessExecutionError(IOError): MESSAGE_TMPL = ('%(description)s\n' @@ -630,7 +643,7 @@ def read_optional_seed(fill, base="", ext="", timeout=5): fill['user-data'] = ud fill['meta-data'] = md return True - except OSError as e: + except IOError as e: if e.errno == errno.ENOENT: return False raise @@ -670,9 +683,12 @@ def fetch_ssl_details(paths=None): def read_file_or_url(url, timeout=5, retries=10, headers=None, data=None, sec_between=1, ssl_details=None): + url = url.lstrip() if url.startswith("/"): url = "file://%s" % url if url.lower().startswith("file://"): + if data: + LOG.warn("Unable to post data to file resource %s", url) file_path = url[len("file://"):] return FileResponse(file_path, contents=load_file(file_path)) else: @@ -724,13 +740,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): md_resp = read_file_or_url(md_url, timeout, retries, file_retries) md = None - if md_resp.ok: + if md_resp.ok(): md_str = str(md_resp) md = load_yaml(md_str, default={}) ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries) ud = None - if ud_resp.ok: + if ud_resp.ok(): ud_str = str(ud_resp) ud = ud_str @@ -900,7 +916,7 @@ def get_cmdline_url(names=('cloud-config-url', 'url'), return (None, None, None) resp = read_file_or_url(url) - if resp.contents.startswith(starts) and resp.ok: + if resp.contents.startswith(starts) and resp.ok(): return (key, url, str(resp)) return (key, url, None) diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index ac082076..d707afa9 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -191,8 +191,8 @@ class TestCmdlineUrl(MockerTestCase): mock_readurl = self.mocker.replace(url_helper.readurl, passthrough=False) - mock_readurl(url) - self.mocker.result(url_helper.UrlResponse(200, payload)) + mock_readurl(url, ARGS, KWARGS) + self.mocker.result(util.StringResponse(payload)) self.mocker.replay() self.assertEqual((key, url, None), @@ -207,8 +207,8 @@ class TestCmdlineUrl(MockerTestCase): mock_readurl = self.mocker.replace(url_helper.readurl, passthrough=False) - mock_readurl(url) - self.mocker.result(url_helper.UrlResponse(200, payload)) + mock_readurl(url, ARGS, KWARGS) + self.mocker.result(util.StringResponse(payload)) self.mocker.replay() self.assertEqual((key, url, payload), @@ -221,7 +221,7 @@ class TestCmdlineUrl(MockerTestCase): cmdline = "ro %s=%s bar=1" % (key, url) self.mocker.replace(url_helper.readurl, passthrough=False) - self.mocker.result(url_helper.UrlResponse(400)) + self.mocker.result(util.StringResponse("")) self.mocker.replay() self.assertEqual((None, None, None), diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index b56fea82..47f8caa4 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -3,12 +3,13 @@ import os from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper +from cloudinit import util from tests.unittests.helpers import populate_dir -from mocker import MockerTestCase +import mocker -class TestMAASDataSource(MockerTestCase): +class TestMAASDataSource(mocker.MockerTestCase): def setUp(self): super(TestMAASDataSource, self).setUp() @@ -115,9 +116,11 @@ class TestMAASDataSource(MockerTestCase): for key in valid_order: url = "%s/%s/%s" % (my_seed, my_ver, key) - mock_request(url, headers=my_headers, timeout=None) + mock_request(url, headers=my_headers, timeout=mocker.ANY, + data=mocker.ANY, sec_between=mocker.ANY, + ssl_details=mocker.ANY, retries=mocker.ANY) resp = valid.get(key) - self.mocker.result(url_helper.UrlResponse(200, resp)) + self.mocker.result(util.StringResponse(resp)) self.mocker.replay() (userdata, metadata) = DataSourceMAAS.read_maas_seed_url(my_seed, From f16b5f79a9bee0f39e5af9ef9ff9049398f11f24 Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 23 Feb 2013 23:22:57 -0800 Subject: [PATCH 11/22] Fix the maas callback mechanism now that requests is used. --- cloudinit/sources/DataSourceMAAS.py | 25 +++++++++---------- cloudinit/url_helper.py | 38 ++++++++++++++++++----------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 0c526305..f3759b4b 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -25,8 +25,6 @@ import os import time import urllib2 -import requests - from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper @@ -81,7 +79,7 @@ class DataSourceMAAS(sources.DataSource): self.base_url = url (userdata, metadata) = read_maas_seed_url(self.base_url, - self.md_headers, + self._md_headers, paths=self.paths) self.userdata_raw = userdata self.metadata = metadata @@ -90,7 +88,7 @@ class DataSourceMAAS(sources.DataSource): util.logexc(LOG, "Failed fetching metadata from url %s", url) return False - def md_headers(self, url): + def _md_headers(self, url): mcfg = self.ds_cfg # If we are missing token_key, token_secret or consumer_key @@ -134,9 +132,10 @@ class DataSourceMAAS(sources.DataSource): starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) urls = [check_url] - url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, exception_cb=self._except_cb, - headers_cb=self.md_headers) + url = url_helper.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, + exception_cb=self._except_cb, + headers_cb=self._md_headers) if url: LOG.debug("Using metadata source: '%s'", url) @@ -147,23 +146,23 @@ class DataSourceMAAS(sources.DataSource): return bool(url) def _except_cb(self, msg, exception): - if not (isinstance(exception, urllib2.HTTPError) and + if not (isinstance(exception, url_helper.UrlError) and (exception.code == 403 or exception.code == 401)): return + if 'date' not in exception.headers: - LOG.warn("date field not in %d headers" % exception.code) + LOG.warn("Missing header 'date' in %s response", exception.code) return date = exception.headers['date'] - try: ret_time = time.mktime(parsedate(date)) - except: - LOG.warn("failed to convert datetime '%s'") + except Exception as e: + LOG.warn("Failed to convert datetime '%s': %s", date, e) return self.oauth_clockskew = int(ret_time - time.time()) - LOG.warn("set oauth clockskew to %d" % self.oauth_clockskew) + LOG.warn("Setting oauth clockskew to %d", self.oauth_clockskew) return diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 300e70c2..6f06761a 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -92,13 +92,13 @@ class UrlResponse(object): class UrlError(IOError): - def __init__(self, cause): + def __init__(self, cause, code=None, headers=None): IOError.__init__(self, str(cause)) self.cause = cause - if isinstance(cause, exceptions.HTTPError) and cause.response: - self.code = cause.response.status_code - else: - self.code = None + self.code = code + self.headers = headers + if self.headers is None: + self.headers = {} def readurl(url, data=None, timeout=None, retries=0, sec_between=1, @@ -170,7 +170,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # attrs return UrlResponse(r) except exceptions.RequestException as e: - excps.append(UrlError(e)) + if isinstance(e, (exceptions.HTTPError)) and e.response: + excps.append(UrlError(e, code=e.response.status_code, + headers=e.response.headers)) + else: + excps.append(UrlError(e)) if i + 1 < manual_tries and sec_between > 0: LOG.debug("Please wait %s seconds while we wait to try again", sec_between) @@ -235,20 +239,23 @@ def wait_for_url(urls, max_wait=None, timeout=None, timeout = int((start_time + max_wait) - now) reason = "" + e = None try: if headers_cb is not None: headers = headers_cb(url) else: headers = {} - resp = readurl(url, headers=headers, timeout=timeout, - check_status=False) - if not resp.contents: - reason = "empty response [%s]" % (resp.code) - e = ValueError(reason) - elif not resp.ok(): - reason = "bad status code [%s]" % (resp.code) - e = ValueError(reason) + response = readurl(url, headers=headers, timeout=timeout, + check_status=False) + if not response.contents: + reason = "empty response [%s]" % (response.code) + e = UrlError(ValueError(reason), + code=response.code, headers=response.headers) + elif not response.ok(): + reason = "bad status code [%s]" % (response.code) + e = UrlError(ValueError(reason), + code=response.code, headers=response.headers) else: return url except UrlError as e: @@ -263,6 +270,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, reason) status_cb(status_msg) if exception_cb: + # This can be used to alter the headers that will be sent + # in the future, for example this is what the MAAS datasource + # does. exception_cb(msg=status_msg, exception=e) if timeup(max_wait, start_time): From 37e6ccff09d1fe5c0b23c8ec41b60c27550b1110 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 13 Mar 2013 22:28:39 -0700 Subject: [PATCH 12/22] Update to handle requests >= 1.0 which doesn't use the config dict. --- cloudinit/url_helper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 6f06761a..08e5f01b 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -34,7 +34,7 @@ LOG = logging.getLogger(__name__) # Check if requests has ssl support (added in requests >= 0.8.8) SSL_ENABLED = False -CONFIG_ENABLED = False # This was added in 0.7 +CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0) try: import pkg_resources from distutils.version import LooseVersion @@ -42,7 +42,7 @@ try: _REQ_VER = LooseVersion(_REQ.version) if _REQ_VER >= LooseVersion('0.8.8'): SSL_ENABLED = True - if _REQ_VER >= LooseVersion('0.7.0'): + if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): CONFIG_ENABLED = True except: pass @@ -129,8 +129,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, if data: req_args['method'] = 'POST' # It doesn't seem like config - # was added in older library versions, thus we - # need to manually do the retries if it wasn't + # was added in older library versions (or newer ones either), thus we + # need to manually do the retries if it wasn't... if CONFIG_ENABLED: req_config = { 'store_cookies': False, From 481a8b04b2e8818942a09f04b6b439608301b6cb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 13 Mar 2013 22:33:18 -0700 Subject: [PATCH 13/22] Fix how the http error doesn't always have the response attached in earlier versions of requests (pre 0.10.8). --- cloudinit/url_helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 08e5f01b..bfc5cfdd 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -170,7 +170,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # attrs return UrlResponse(r) except exceptions.RequestException as e: - if isinstance(e, (exceptions.HTTPError)) and e.response: + if (isinstance(e, (exceptions.HTTPError)) + and hasattr(e, 'response') # This appeared in v 0.10.8 + and e.response): excps.append(UrlError(e, code=e.response.status_code, headers=e.response.headers)) else: From 505519873226b4a5b4a832bf1c20f7158933c341 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 19 Mar 2013 09:32:04 -0400 Subject: [PATCH 14/22] appease pylint and pep8 * cloudinit/distros/parsers/resolv_conf.py added some pylint overrides with 'plXXXXX' syntax. example: # pl51222 pylint: disable=E0102 The pl51222 there means: http://www.logilab.org/ticket/51222 This specific issue is present in 12.04 pylint, but not 13.04. * pylint doesn't like the requests special handling we have. which makes sense as it is only checking versus one specific version. * general pep8 and pylint cleanups. --- cloudinit/distros/parsers/resolv_conf.py | 4 ++-- cloudinit/ec2_utils.py | 1 + cloudinit/url_helper.py | 22 ++++++++++++---------- cloudinit/util.py | 4 ++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 5733c25a..1be9d46b 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -137,8 +137,8 @@ class ResolvConf(object): self._contents.append(('option', ['search', s_list, ''])) return flat_sds - @local_domain.setter - def local_domain(self, domain): + @local_domain.setter # pl51222 pylint: disable=E1101 + def local_domain(self, domain): # pl51222 pylint: disable=E0102 self.parse() self._remove_option('domain') self._contents.append(('option', ['domain', str(domain), ''])) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 4812eccb..6b2754aa 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -145,6 +145,7 @@ def get_instance_userdata(api_version, metadata_address, ssl_details=None): util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return None + def get_instance_metadata(api_version, metadata_address, ssl_details=None): md_url = combine_url(metadata_address, api_version) md_url = combine_url(md_url, 'meta-data') diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index bfc5cfdd..de73cc84 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -34,12 +34,12 @@ LOG = logging.getLogger(__name__) # Check if requests has ssl support (added in requests >= 0.8.8) SSL_ENABLED = False -CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0) +CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0) try: - import pkg_resources from distutils.version import LooseVersion + import pkg_resources _REQ = pkg_resources.get_distribution('requests') - _REQ_VER = LooseVersion(_REQ.version) + _REQ_VER = LooseVersion(_REQ.version) # pylint: disable=E1103 if _REQ_VER >= LooseVersion('0.8.8'): SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): @@ -49,7 +49,7 @@ except: def _cleanurl(url): - parsed_url = list(urlparse(url, scheme='http')) + parsed_url = list(urlparse(url, scheme='http')) # pylint: disable=E1123 if not parsed_url[1] and parsed_url[2]: # Swap these since this seems to be a common # occurrence when given urls like 'www.google.com' @@ -108,7 +108,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, req_args = { 'url': url, } - if urlparse(url).scheme == 'https' and ssl_details: + scheme = urlparse(url).scheme # pylint: disable=E1101 + if scheme == 'https' and ssl_details: if not SSL_ENABLED: LOG.warn("SSL is not enabled, cert. verification can not occur!") else: @@ -121,7 +122,7 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, ssl_details['key_file']] elif 'cert_file' in ssl_details: req_args['cert'] = str(ssl_details['cert_file']) - + req_args['allow_redirects'] = allow_redirects req_args['method'] = 'GET' if timeout is not None: @@ -162,16 +163,17 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, try: r = requests.request(**req_args) if check_status: - r.raise_for_status() + r.raise_for_status() # pylint: disable=E1103 LOG.debug("Read from %s (%s, %sb) after %s attempts", url, - r.status_code, len(r.content), (i + 1)) + r.status_code, len(r.content), # pylint: disable=E1103 + (i + 1)) # Doesn't seem like we can make it use a different # subclass for responses, so add our own backward-compat # attrs return UrlResponse(r) except exceptions.RequestException as e: if (isinstance(e, (exceptions.HTTPError)) - and hasattr(e, 'response') # This appeared in v 0.10.8 + and hasattr(e, 'response') # This appeared in v 0.10.8 and e.response): excps.append(UrlError(e, code=e.response.status_code, headers=e.response.headers)) @@ -183,7 +185,7 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, time.sleep(sec_between) if excps: raise excps[-1] - return None # Should throw before this... + return None # Should throw before this... def wait_for_url(urls, max_wait=None, timeout=None, diff --git a/cloudinit/util.py b/cloudinit/util.py index 52b528ea..36e9b83b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -51,8 +51,8 @@ from cloudinit import importer from cloudinit import log as logging from cloudinit import mergers from cloudinit import safeyaml -from cloudinit import url_helper from cloudinit import type_utils +from cloudinit import url_helper from cloudinit import version from cloudinit.settings import (CFG_BUILTIN) @@ -81,7 +81,7 @@ class StringResponse(object): self.contents = contents self.url = None - def ok(self, *args, **kwargs): + def ok(self, *args, **kwargs): # pylint: disable=W0613 if self.code != 200: return False return True From 89bc8a2dfca6bf47e5a072b695b54e2716c29ef0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 19 Mar 2013 09:51:02 -0400 Subject: [PATCH 15/22] do not bother retrying on ssl errors if the error is an ssl error, its extremely unlikely that it would be fixed by waiting a few seconds and trying again. --- cloudinit/url_helper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index de73cc84..ac6f25db 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -179,6 +179,10 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=e.response.headers)) else: excps.append(UrlError(e)) + if SSL_ENABLED and isinstance(e, exceptions.SSLError): + # ssl exceptions are not going to get fixed by waiting a + # few seconds + break if i + 1 < manual_tries and sec_between > 0: LOG.debug("Please wait %s seconds while we wait to try again", sec_between) From 4ab16c490e9a8b900e4b7e2b96ad89d8f4923ea9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 19 Mar 2013 10:05:51 -0400 Subject: [PATCH 16/22] set 'allow_redirects' to True by default the previous implementation of url_helper.readurl() would default to allow_redirects being true. So, for backwards compat, we should keep that behavior. --- cloudinit/url_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index ac6f25db..6b4516e0 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -103,7 +103,7 @@ class UrlError(IOError): def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, ssl_details=None, check_status=True, - allow_redirects=False): + allow_redirects=True): url = _cleanurl(url) req_args = { 'url': url, From edc25679959e956a15d641f18e2ecc55628982af Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 19 Mar 2013 10:35:13 -0400 Subject: [PATCH 17/22] make get_instance_userdata and get_instance_metadata more like botos this shouldn't change anything, only the signatures of the methods. --- cloudinit/ec2_utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 6b2754aa..87644c40 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -135,8 +135,9 @@ class MetadataMaterializer(object): return joined -def get_instance_userdata(api_version, metadata_address, ssl_details=None): - ud_url = combine_url(metadata_address, api_version) +def get_instance_userdata(version='latest', url='http://169.254.169.254', + ssl_details=None): + ud_url = combine_url(url, version) ud_url = combine_url(ud_url, 'user-data') try: response = util.read_file_or_url(ud_url, ssl_details=ssl_details) @@ -146,11 +147,13 @@ def get_instance_userdata(api_version, metadata_address, ssl_details=None): return None -def get_instance_metadata(api_version, metadata_address, ssl_details=None): - md_url = combine_url(metadata_address, api_version) +def get_instance_metadata(version='latest', url='http://169.254.169.254', + ssl_details=None, timeout=5, num_retries=5) + md_url = combine_url(metadata_address, version) md_url = combine_url(md_url, 'meta-data') try: - response = util.read_file_or_url(md_url, ssl_details=ssl_details) + response = util.read_file_or_url(md_url, ssl_details=ssl_details, + timeout=timeout, retries=num_retries) materializer = MetadataMaterializer(str(response), md_url, ssl_details) return materializer.materialize() except Exception: From e52c47f60fd74f873b953be86019add3a4986573 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 19 Mar 2013 11:12:59 -0400 Subject: [PATCH 18/22] fix typo --- cloudinit/ec2_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 87644c40..71c84206 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -148,7 +148,7 @@ def get_instance_userdata(version='latest', url='http://169.254.169.254', def get_instance_metadata(version='latest', url='http://169.254.169.254', - ssl_details=None, timeout=5, num_retries=5) + ssl_details=None, timeout=5, num_retries=5): md_url = combine_url(metadata_address, version) md_url = combine_url(md_url, 'meta-data') try: From e82f3008419b38abe9d4194863392948ee7127ec Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Mar 2013 15:53:16 -0700 Subject: [PATCH 19/22] Move back to using boto for now. --- Requires | 3 + cloudinit/ec2_utils.py | 169 +++++----------------- cloudinit/sources/DataSourceCloudStack.py | 7 +- cloudinit/sources/DataSourceEc2.py | 7 +- 4 files changed, 41 insertions(+), 145 deletions(-) diff --git a/Requires b/Requires index 0313d569..de51a4e4 100644 --- a/Requires +++ b/Requires @@ -24,3 +24,6 @@ argparse # Requests handles ssl correctly! requests + +# Boto for ec2 +boto diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 71c84206..29393ce1 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,146 +16,45 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from urlparse import (urlparse, urlunparse) +import boto.utils as boto_utils -import json -import urllib - -from cloudinit import log as logging -from cloudinit import util - -LOG = logging.getLogger(__name__) +# Versions of boto >= 2.6.0 (and possibly 2.5.2) +# try to lazily load the metadata backing, which +# doesn't work so well in cloud-init especially +# since the metadata is serialized and actions are +# performed where the metadata server may be blocked +# (thus the datasource will start failing) resulting +# in url exceptions when fields that do exist (or +# would have existed) do not exist due to the blocking +# that occurred. -def combine_url(base, add_on): - base_parsed = list(urlparse(base)) - path = base_parsed[2] - if path and not path.endswith("/"): - path += "/" - path += urllib.quote(str(add_on), safe="/:") - base_parsed[2] = path - return urlunparse(base_parsed) +def _unlazy_dict(mp): + if not isinstance(mp, (dict)): + return mp + # Walk over the keys/values which + # forces boto to unlazy itself and + # has no effect on dictionaries that + # already have there items. + for (_k, v) in mp.items(): + _unlazy_dict(v) + return mp -# See: http://bit.ly/TyoUQs -# -# Since boto metadata reader uses the old urllib which does not -# support ssl, we need to ahead and create our own reader which -# works the same as the boto one (for now). -class MetadataMaterializer(object): - def __init__(self, blob, base_url, ssl_details): - self._blob = blob - self._md = None - self._base_url = base_url - self._ssl_details = ssl_details - - def _parse(self, blob): - leaves = {} - children = [] - if not blob: - return (leaves, children) - - def has_children(item): - if item.endswith("/"): - return True - else: - return False - - def get_name(item): - if item.endswith("/"): - return item.rstrip("/") - return item - - for field in blob.splitlines(): - field = field.strip() - field_name = get_name(field) - if not field or not field_name: - continue - if has_children(field): - if field_name not in children: - children.append(field_name) - else: - contents = field.split("=", 1) - resource = field_name - if len(contents) > 1: - # What a PITA... - (ident, sub_contents) = contents - checked_ident = util.safe_int(ident) - if checked_ident is not None: - resource = "%s/openssh-key" % (checked_ident) - field_name = sub_contents - leaves[field_name] = resource - return (leaves, children) - - def materialize(self): - if self._md is not None: - return self._md - self._md = self._materialize(self._blob, self._base_url) - return self._md - - def _fetch_url(self, url): - response = util.read_file_or_url(url, ssl_details=self._ssl_details) - return str(response) - - def _decode_leaf_blob(self, blob): - if not blob: - return blob - stripped_blob = blob.strip() - if stripped_blob.startswith("{") and stripped_blob.endswith("}"): - # Assume and try with json - try: - return json.loads(blob) - except (ValueError, TypeError): - pass - if blob.find("\n") != -1: - return blob.splitlines() - return blob - - def _materialize(self, blob, base_url): - (leaves, children) = self._parse(blob) - child_contents = {} - for c in children: - child_url = combine_url(base_url, c) - if not child_url.endswith("/"): - child_url += "/" - child_blob = self._fetch_url(child_url) - child_contents[c] = self._materialize(child_blob, child_url) - leaf_contents = {} - for (field, resource) in leaves.items(): - leaf_url = combine_url(base_url, resource) - leaf_blob = self._fetch_url(leaf_url) - leaf_contents[field] = self._decode_leaf_blob(leaf_blob) - joined = {} - joined.update(child_contents) - for field in leaf_contents.keys(): - if field in joined: - LOG.warn("Duplicate key found in results from %s", base_url) - else: - joined[field] = leaf_contents[field] - return joined +def get_instance_userdata(api_version, metadata_address): + # Note: boto.utils.get_instance_metadata returns '' for empty string + # so the change from non-true to '' is not specifically necessary, but + # this way cloud-init will get consistent behavior even if boto changed + # in the future to return a None on "no user-data provided". + ud = boto_utils.get_instance_userdata(api_version, None, metadata_address) + if not ud: + ud = '' + return ud -def get_instance_userdata(version='latest', url='http://169.254.169.254', - ssl_details=None): - ud_url = combine_url(url, version) - ud_url = combine_url(ud_url, 'user-data') - try: - response = util.read_file_or_url(ud_url, ssl_details=ssl_details) - return str(response) - except Exception: - util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) - return None +def get_instance_metadata(api_version, metadata_address): + metadata = boto_utils.get_instance_metadata(api_version, metadata_address) + if not isinstance(metadata, (dict)): + metadata = {} + return _unlazy_dict(metadata) - -def get_instance_metadata(version='latest', url='http://169.254.169.254', - ssl_details=None, timeout=5, num_retries=5): - md_url = combine_url(metadata_address, version) - md_url = combine_url(md_url, 'meta-data') - try: - response = util.read_file_or_url(md_url, ssl_details=ssl_details, - timeout=timeout, retries=num_retries) - materializer = MetadataMaterializer(str(response), md_url, ssl_details) - return materializer.materialize() - except Exception: - util.logexc(LOG, "Failed fetching metadata from url %s", md_url) - return None diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 7fd38982..b4ca6d93 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -102,13 +102,10 @@ class DataSourceCloudStack(sources.DataSource): return False start_time = time.time() md_addr = self.metadata_address - ssl_details = util.fetch_ssl_details(self.paths) self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr, - ssl_details) + md_addr) self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr, - ssl_details) + md_addr) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 0d62cf01..bd35c8b0 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -62,13 +62,10 @@ class DataSourceEc2(sources.DataSource): return False start_time = time.time() md_addr = self.metadata_address - ssl_details = util.fetch_ssl_details(self.paths) self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr, - ssl_details) + md_addr) self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr, - ssl_details) + md_addr) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True From 4aef583c639f628a36d528c0399134be5b53f626 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Mar 2013 15:55:58 -0700 Subject: [PATCH 20/22] Add doc about issue 1401 in boto. --- cloudinit/ec2_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 29393ce1..175f96aa 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -28,6 +28,10 @@ import boto.utils as boto_utils # would have existed) do not exist due to the blocking # that occurred. +# TODO: https://github.com/boto/boto/issues/1401 +# When boto finally moves to using requests, we should be able +# to provide it ssl details, it does not yet, so we can't provide them... + def _unlazy_dict(mp): if not isinstance(mp, (dict)): From e382bdee21a2fbfbc064a7c76e179a86024a8b60 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 20 Mar 2013 08:30:43 -0400 Subject: [PATCH 21/22] remove some churn cloudinit/sources/DataSourceCloudStack.py and cloudinit/sources/DataSourceEc2.py are reverted entirely back to trunk versions now, rather than the non-behavior change that was left in place. Also, remove inadvertantly added trailing newline from cloudinit/ec2_utils.py Overall, this just makes the diff when merged to trunk carry more focused changes. --- cloudinit/ec2_utils.py | 1 - cloudinit/sources/DataSourceCloudStack.py | 11 +++++------ cloudinit/sources/DataSourceEc2.py | 13 ++++++------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 175f96aa..5fa8c5ad 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -61,4 +61,3 @@ def get_instance_metadata(api_version, metadata_address): if not isinstance(metadata, (dict)): metadata = {} return _unlazy_dict(metadata) - diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index b4ca6d93..81c8cda9 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -25,7 +25,7 @@ import os import time -from cloudinit import ec2_utils +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -101,11 +101,10 @@ class DataSourceCloudStack(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - md_addr = self.metadata_address - self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr) - self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index bd35c8b0..f010e640 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -23,7 +23,7 @@ import os import time -from cloudinit import ec2_utils +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -61,11 +61,10 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - md_addr = self.metadata_address - self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr) - self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True @@ -134,7 +133,7 @@ class DataSourceEc2(sources.DataSource): start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url2base[url]) From 7ca654ea32c00c638039895c1c15dc062676900b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 20 Mar 2013 08:34:19 -0400 Subject: [PATCH 22/22] pep8 --- cloudinit/ec2_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 5fa8c5ad..fcd511c5 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -28,7 +28,7 @@ import boto.utils as boto_utils # would have existed) do not exist due to the blocking # that occurred. -# TODO: https://github.com/boto/boto/issues/1401 +# TODO(harlowja): https://github.com/boto/boto/issues/1401 # When boto finally moves to using requests, we should be able # to provide it ssl details, it does not yet, so we can't provide them...