Bring back the ec2 utils, non-boto userdata/metadata reading

This replacement uses our own userdata/metadata ec2 webservice
parser that we can easily modify, it also automatically allows
for reading the ec2 userdata/metdata from files and also brings
in the usage of requests instead of boto's usage of urllib which
did not support ssl properly.

cloudinit/ec2_utils.py

@@ -16,48 +16,154 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import boto.utils as boto_utils
+from urlparse import (urlparse, urlunparse)
 
-# Versions of boto >= 2.6.0 (and possibly 2.5.2)
+import functools
-# try to lazily load the metadata backing, which
+import json
-# doesn't work so well in cloud-init especially
+import urllib
-# since the metadata is serialized and actions are
-# performed where the metadata server may be blocked
-# (thus the datasource will start failing) resulting
-# in url exceptions when fields that do exist (or
-# would have existed) do not exist due to the blocking
-# that occurred.
 
-# TODO(harlowja): https://github.com/boto/boto/issues/1401
+from cloudinit import log as logging
-# When boto finally moves to using requests, we should be able
+from cloudinit import util
-# to provide it ssl details, it does not yet, so we can't provide them...
+
+LOG = logging.getLogger(__name__)
 
 
-def _unlazy_dict(mp):
+def combine_url(base, add_on):
-    if not isinstance(mp, (dict)):
+    base_parsed = list(urlparse(base))
-        return mp
+    path = base_parsed[2]
-    # Walk over the keys/values which
+    if path and not path.endswith("/"):
-    # forces boto to unlazy itself and
+        path += "/"
-    # has no effect on dictionaries that
+    path += urllib.quote(str(add_on), safe="/:")
-    # already have there items.
+    base_parsed[2] = path
-    for (_k, v) in mp.items():
+    return urlunparse(base_parsed)
-        _unlazy_dict(v)
-    return mp
 
 
-def get_instance_userdata(api_version, metadata_address):
+# See: http://bit.ly/TyoUQs
-    # Note: boto.utils.get_instance_metadata returns '' for empty string
+#
-    # so the change from non-true to '' is not specifically necessary, but
+# Since boto metadata reader uses the old urllib which does not
-    # this way cloud-init will get consistent behavior even if boto changed
+# support ssl, we need to ahead and create our own reader which
-    # in the future to return a None on "no user-data provided".
+# works the same as the boto one (for now).
-    ud = boto_utils.get_instance_userdata(api_version, None, metadata_address)
+class MetadataMaterializer(object):
-    if not ud:
+    def __init__(self, blob, base_url, caller):
-        ud = ''
+        self._blob = blob
-    return ud
+        self._md = None
+        self._base_url = base_url
+        self._caller = caller
+
+    def _parse(self, blob):
+        leaves = {}
+        children = []
+        if not blob:
+            return (leaves, children)
+
+        def has_children(item):
+            if item.endswith("/"):
+                return True
+            else:
+                return False
+
+        def get_name(item):
+            if item.endswith("/"):
+                return item.rstrip("/")
+            return item
+
+        for field in blob.splitlines():
+            field = field.strip()
+            field_name = get_name(field)
+            if not field or not field_name:
+                continue
+            if has_children(field):
+                if field_name not in children:
+                    children.append(field_name)
+            else:
+                contents = field.split("=", 1)
+                resource = field_name
+                if len(contents) > 1:
+                    # What a PITA...
+                    (ident, sub_contents) = contents
+                    checked_ident = util.safe_int(ident)
+                    if checked_ident is not None:
+                        resource = "%s/openssh-key" % (checked_ident)
+                        field_name = sub_contents
+                leaves[field_name] = resource
+        return (leaves, children)
+
+    def materialize(self):
+        if self._md is not None:
+            return self._md
+        self._md = self._materialize(self._blob, self._base_url)
+        return self._md
+
+    def _decode_leaf_blob(self, blob):
+        if not blob:
+            return blob
+        stripped_blob = blob.strip()
+        if stripped_blob.startswith("{") and stripped_blob.endswith("}"):
+            # Assume and try with json
+            try:
+                return json.loads(blob)
+            except (ValueError, TypeError):
+                pass
+        if blob.find("\n") != -1:
+            return blob.splitlines()
+        return blob
+
+    def _materialize(self, blob, base_url):
+        (leaves, children) = self._parse(blob)
+        child_contents = {}
+        for c in children:
+            child_url = combine_url(base_url, c)
+            if not child_url.endswith("/"):
+                child_url += "/"
+            child_blob = str(self._caller(child_url))
+            child_contents[c] = self._materialize(child_blob, child_url)
+        leaf_contents = {}
+        for (field, resource) in leaves.items():
+            leaf_url = combine_url(base_url, resource)
+            leaf_blob = str(self._caller(leaf_url))
+            leaf_contents[field] = self._decode_leaf_blob(leaf_blob)
+        joined = {}
+        joined.update(child_contents)
+        for field in leaf_contents.keys():
+            if field in joined:
+                LOG.warn("Duplicate key found in results from %s", base_url)
+            else:
+                joined[field] = leaf_contents[field]
+        return joined
 
 
-def get_instance_metadata(api_version, metadata_address):
+def get_instance_userdata(api_version='latest',
-    metadata = boto_utils.get_instance_metadata(api_version, metadata_address)
+                          metadata_address='http://169.254.169.254',
-    if not isinstance(metadata, (dict)):
+                          ssl_details=None, timeout=5, retries=5):
-        metadata = {}
+    ud_url = combine_url(metadata_address, api_version)
-    return _unlazy_dict(metadata)
+    ud_url = combine_url(ud_url, 'user-data')
+    try:
+        response = util.read_file_or_url(ud_url,
+                                         ssl_details=ssl_details,
+                                         timeout=timeout,
+                                         retries=retries)
+        return str(response)
+    except Exception:
+        util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
+        return None
+
+
+def get_instance_metadata(api_version='latest',
+                          metadata_address='http://169.254.169.254',
+                          ssl_details=None, timeout=5, retries=5):
+    md_url = combine_url(metadata_address, api_version)
+    md_url = combine_url(md_url, 'meta-data')
+    caller = functools.partial(util.read_file_or_url,
+                               ssl_details=ssl_details, timeout=timeout,
+                               retries=retries)
+
+    try:
+        response = caller(md_url)
+        materializer = MetadataMaterializer(str(response), md_url, caller)
+        md = materializer.materialize()
+        if not isinstance(md, (dict)):
+            md = {}
+        return md
+    except Exception:
+        util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
+        return {}