drop requirement on boto for its boto.utils.get_instance_metadata()

We had a requirement on boto only to use
boto.utils.get_instance_metadata().  That had actually caused some pain in
the past.  This removes a Requires and also one that wasn't python3.

ChangeLog

@@ -14,6 +14,7 @@
  - drop support for resizing partitions with parted entirely (LP: #1212492).
    This was broken as it was anyway.
  - add support for vendordata.
+ - drop dependency on boto for crawling ec2 metadata service.
 0.7.4:
  - fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a
    partitioned block device with target filesystem on ephemeral0.1.

Requires

@@ -29,8 +29,8 @@ argparse
 # Requests handles ssl correctly!
 requests
 
-# Boto for ec2
-boto
-
 # For patching pieces of cloud-config together
 jsonpatch
+
+# For http testing (only needed for testing)
+httpretty>=0.7.1

cloudinit/ec2_utils.py

@@ -16,48 +16,163 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import boto.utils as boto_utils
+from urlparse import (urlparse, urlunparse)
 
-# Versions of boto >= 2.6.0 (and possibly 2.5.2)
+import functools
-# try to lazily load the metadata backing, which
+import json
-# doesn't work so well in cloud-init especially
+import urllib
-# since the metadata is serialized and actions are
-# performed where the metadata server may be blocked
-# (thus the datasource will start failing) resulting
-# in url exceptions when fields that do exist (or
-# would have existed) do not exist due to the blocking
-# that occurred.
 
-# TODO(harlowja): https://github.com/boto/boto/issues/1401
+from cloudinit import log as logging
-# When boto finally moves to using requests, we should be able
+from cloudinit import util
-# to provide it ssl details, it does not yet, so we can't provide them...
+
+LOG = logging.getLogger(__name__)
 
 
-def _unlazy_dict(mp):
+def maybe_json_object(text):
-    if not isinstance(mp, (dict)):
+    if not text:
-        return mp
+        return False
-    # Walk over the keys/values which
+    text = text.strip()
-    # forces boto to unlazy itself and
+    if text.startswith("{") and text.endswith("}"):
-    # has no effect on dictionaries that
+        return True
-    # already have there items.
+    return False
-    for (_k, v) in mp.items():
-        _unlazy_dict(v)
-    return mp
 
 
-def get_instance_userdata(api_version, metadata_address):
+def combine_url(base, add_on):
-    # Note: boto.utils.get_instance_metadata returns '' for empty string
+    base_parsed = list(urlparse(base))
-    # so the change from non-true to '' is not specifically necessary, but
+    path = base_parsed[2]
-    # this way cloud-init will get consistent behavior even if boto changed
+    if path and not path.endswith("/"):
-    # in the future to return a None on "no user-data provided".
+        path += "/"
-    ud = boto_utils.get_instance_userdata(api_version, None, metadata_address)
+    path += urllib.quote(str(add_on), safe="/:")
-    if not ud:
+    base_parsed[2] = path
-        ud = ''
+    return urlunparse(base_parsed)
-    return ud
 
 
-def get_instance_metadata(api_version, metadata_address):
+# See: http://bit.ly/TyoUQs
-    metadata = boto_utils.get_instance_metadata(api_version, metadata_address)
+#
-    if not isinstance(metadata, (dict)):
+# Since boto metadata reader uses the old urllib which does not
-        metadata = {}
+# support ssl, we need to ahead and create our own reader which
-    return _unlazy_dict(metadata)
+# works the same as the boto one (for now).
+class MetadataMaterializer(object):
+    def __init__(self, blob, base_url, caller):
+        self._blob = blob
+        self._md = None
+        self._base_url = base_url
+        self._caller = caller
+
+    def _parse(self, blob):
+        leaves = {}
+        children = []
+        if not blob:
+            return (leaves, children)
+
+        def has_children(item):
+            if item.endswith("/"):
+                return True
+            else:
+                return False
+
+        def get_name(item):
+            if item.endswith("/"):
+                return item.rstrip("/")
+            return item
+
+        for field in blob.splitlines():
+            field = field.strip()
+            field_name = get_name(field)
+            if not field or not field_name:
+                continue
+            if has_children(field):
+                if field_name not in children:
+                    children.append(field_name)
+            else:
+                contents = field.split("=", 1)
+                resource = field_name
+                if len(contents) > 1:
+                    # What a PITA...
+                    (ident, sub_contents) = contents
+                    ident = util.safe_int(ident)
+                    if ident is not None:
+                        resource = "%s/openssh-key" % (ident)
+                        field_name = sub_contents
+                leaves[field_name] = resource
+        return (leaves, children)
+
+    def materialize(self):
+        if self._md is not None:
+            return self._md
+        self._md = self._materialize(self._blob, self._base_url)
+        return self._md
+
+    def _decode_leaf_blob(self, field, blob):
+        if not blob:
+            return blob
+        if maybe_json_object(blob):
+            try:
+                # Assume it's json, unless it fails parsing...
+                return json.loads(blob)
+            except (ValueError, TypeError) as e:
+                LOG.warn("Field %s looked like a json object, but it was"
+                         " not: %s", field, e)
+        if blob.find("\n") != -1:
+            return blob.splitlines()
+        return blob
+
+    def _materialize(self, blob, base_url):
+        (leaves, children) = self._parse(blob)
+        child_contents = {}
+        for c in children:
+            child_url = combine_url(base_url, c)
+            if not child_url.endswith("/"):
+                child_url += "/"
+            child_blob = str(self._caller(child_url))
+            child_contents[c] = self._materialize(child_blob, child_url)
+        leaf_contents = {}
+        for (field, resource) in leaves.items():
+            leaf_url = combine_url(base_url, resource)
+            leaf_blob = str(self._caller(leaf_url))
+            leaf_contents[field] = self._decode_leaf_blob(field, leaf_blob)
+        joined = {}
+        joined.update(child_contents)
+        for field in leaf_contents.keys():
+            if field in joined:
+                LOG.warn("Duplicate key found in results from %s", base_url)
+            else:
+                joined[field] = leaf_contents[field]
+        return joined
+
+
+def get_instance_userdata(api_version='latest',
+                          metadata_address='http://169.254.169.254',
+                          ssl_details=None, timeout=5, retries=5):
+    ud_url = combine_url(metadata_address, api_version)
+    ud_url = combine_url(ud_url, 'user-data')
+    try:
+        response = util.read_file_or_url(ud_url,
+                                         ssl_details=ssl_details,
+                                         timeout=timeout,
+                                         retries=retries)
+        return str(response)
+    except Exception:
+        util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
+        return ''
+
+
+def get_instance_metadata(api_version='latest',
+                          metadata_address='http://169.254.169.254',
+                          ssl_details=None, timeout=5, retries=5):
+    md_url = combine_url(metadata_address, api_version)
+    md_url = combine_url(md_url, 'meta-data')
+    caller = functools.partial(util.read_file_or_url,
+                               ssl_details=ssl_details, timeout=timeout,
+                               retries=retries)
+
+    try:
+        response = caller(md_url)
+        materializer = MetadataMaterializer(str(response), md_url, caller)
+        md = materializer.materialize()
+        if not isinstance(md, (dict)):
+            md = {}
+        return md
+    except Exception:
+        util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
+        return {}

tests/unittests/test_ec2_util.py

@@ -0,0 +1,130 @@
+from tests.unittests import helpers
+
+from cloudinit import ec2_utils as eu
+
+import httpretty as hp
+
+
+class TestEc2Util(helpers.TestCase):
+    VERSION = 'latest'
+
+    @hp.activate
+    def test_userdata_fetch(self):
+        hp.register_uri(hp.GET,
+                        'http://169.254.169.254/%s/user-data' % (self.VERSION),
+                        body='stuff',
+                        status=200)
+        userdata = eu.get_instance_userdata(self.VERSION)
+        self.assertEquals('stuff', userdata)
+
+    @hp.activate
+    def test_userdata_fetch_fail_not_found(self):
+        hp.register_uri(hp.GET,
+                        'http://169.254.169.254/%s/user-data' % (self.VERSION),
+                        status=404)
+        userdata = eu.get_instance_userdata(self.VERSION, retries=0)
+        self.assertEquals('', userdata)
+
+    @hp.activate
+    def test_userdata_fetch_fail_server_dead(self):
+        hp.register_uri(hp.GET,
+                        'http://169.254.169.254/%s/user-data' % (self.VERSION),
+                        status=500)
+        userdata = eu.get_instance_userdata(self.VERSION, retries=0)
+        self.assertEquals('', userdata)
+
+    @hp.activate
+    def test_metadata_fetch_no_keys(self):
+        base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
+        hp.register_uri(hp.GET, base_url, status=200,
+                        body="\n".join(['hostname',
+                                        'instance-id',
+                                        'ami-launch-index']))
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+                        status=200, body='ec2.fake.host.name.com')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+                        status=200, body='123')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'ami-launch-index'),
+                        status=200, body='1')
+        md = eu.get_instance_metadata(self.VERSION, retries=0)
+        self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
+        self.assertEquals(md['instance-id'], '123')
+        self.assertEquals(md['ami-launch-index'], '1')
+
+    @hp.activate
+    def test_metadata_fetch_key(self):
+        base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
+        hp.register_uri(hp.GET, base_url, status=200,
+                        body="\n".join(['hostname',
+                                        'instance-id',
+                                        'public-keys/']))
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+                        status=200, body='ec2.fake.host.name.com')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+                        status=200, body='123')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'),
+                        status=200, body='0=my-public-key')
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url, 'public-keys/0/openssh-key'),
+                        status=200, body='ssh-rsa AAAA.....wZEf my-public-key')
+        md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
+        self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
+        self.assertEquals(md['instance-id'], '123')
+        self.assertEquals(1, len(md['public-keys']))
+
+    @hp.activate
+    def test_metadata_fetch_with_2_keys(self):
+        base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
+        hp.register_uri(hp.GET, base_url, status=200,
+                        body="\n".join(['hostname',
+                                        'instance-id',
+                                        'public-keys/']))
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+                        status=200, body='ec2.fake.host.name.com')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+                        status=200, body='123')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'),
+                        status=200,
+                        body="\n".join(['0=my-public-key', '1=my-other-key']))
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url, 'public-keys/0/openssh-key'),
+                        status=200, body='ssh-rsa AAAA.....wZEf my-public-key')
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url, 'public-keys/1/openssh-key'),
+                        status=200, body='ssh-rsa AAAA.....wZEf my-other-key')
+        md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
+        self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
+        self.assertEquals(md['instance-id'], '123')
+        self.assertEquals(2, len(md['public-keys']))
+
+    @hp.activate
+    def test_metadata_fetch_bdm(self):
+        base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
+        hp.register_uri(hp.GET, base_url, status=200,
+                        body="\n".join(['hostname',
+                                        'instance-id',
+                                        'block-device-mapping/']))
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+                        status=200, body='ec2.fake.host.name.com')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+                        status=200, body='123')
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url, 'block-device-mapping/'),
+                        status=200,
+                        body="\n".join(['ami', 'ephemeral0']))
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url, 'block-device-mapping/ami'),
+                        status=200,
+                        body="sdb")
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url,
+                                       'block-device-mapping/ephemeral0'),
+                        status=200,
+                        body="sdc")
+        md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
+        self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
+        self.assertEquals(md['instance-id'], '123')
+        bdm = md['block-device-mapping']
+        self.assertEquals(2, len(bdm))
+        self.assertEquals(bdm['ami'], 'sdb')
+        self.assertEquals(bdm['ephemeral0'], 'sdc')