Remove HEAD usage and other small adjustments

cloudinit/ec2_utils.py

@@ -16,10 +16,8 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import httplib
-from urlparse import (urlparse, urlunparse)
-
 import functools
+import httplib
 import json
 
 from cloudinit import log as logging

cloudinit/sources/DataSourceOpenStack.py

@@ -44,6 +44,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         self.ssl_details = util.fetch_ssl_details(self.paths)
         self.version = None
         self.files = {}
+        self.ec2_metadata = None
 
     def __str__(self):
         root = sources.DataSource.__str__(self)

cloudinit/sources/helpers/openstack.py

@@ -21,7 +21,6 @@
 import abc
 import base64
 import copy
-import functools
 import os
 
 from cloudinit import ec2_utils
@@ -395,26 +394,38 @@ class ConfigDriveReader(BaseReader):
 class MetadataReader(BaseReader):
     def __init__(self, base_url, ssl_details=None, timeout=5, retries=5):
         super(MetadataReader, self).__init__(base_url)
-        self._url_reader = functools.partial(url_helper.readurl,
-                                             retries=retries,
-                                             ssl_details=ssl_details,
-                                             timeout=timeout)
-        self._url_checker = functools.partial(url_helper.existsurl,
-                                              ssl_details=ssl_details,
-                                              timeout=timeout)
-        self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata,
-                                             ssl_details=ssl_details,
-                                             timeout=timeout,
-                                             retries=retries)
+        self.ssl_details = ssl_details
+        self.timeout = float(timeout)
+        self.retries = int(retries)
 
     def _path_read(self, path):
-        return str(self._url_reader(path))
+        response = url_helper.readurl(path,
+                                      retries=self.retries,
+                                      ssl_details=self.ssl_details,
+                                      timeout=self.timeout)
+        return response.contents
 
     def _path_exists(self, path):
-        return self._url_checker(path)
+
+        def should_retry_cb(request, cause):
+            if cause.code >= 400:
+                return False
+            return True
+
+        try:
+            response = url_helper.readurl(path,
+                                          retries=self.retries,
+                                          ssl_details=self.ssl_details,
+                                          timeout=self.timeout,
+                                          exception_cb=should_retry_cb)
+            return response.ok()
+        except IOError:
+            return False
 
     def _path_join(self, base, *add_ons):
         return url_helper.combine_url(base, *add_ons)
 
     def _read_ec2_metadata(self):
-        return self._ec2_reader()
+        return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details,
+                                               timeout=self.timeout,
+                                               retries=self.retries)

cloudinit/url_helper.py

@@ -166,35 +166,16 @@ def _get_ssl_args(url, ssl_details):
     return ssl_args
 
 
-def existsurl(url, ssl_details=None, timeout=None):
-    r = _readurl(url, ssl_details=ssl_details, timeout=timeout,
-                 method='HEAD', check_status=False)
-    return r.ok()
-
-
 def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
-            headers=None, headers_cb=None, ssl_details=None,
-            check_status=True, allow_redirects=True, exception_cb=None):
-    return _readurl(url, data=data, timeout=timeout, retries=retries,
-                    sec_between=sec_between, headers=headers,
-                    headers_cb=headers_cb, ssl_details=ssl_details,
-                    check_status=check_status,
-                    allow_redirects=allow_redirects,
-                    exception_cb=exception_cb)
-
-
-def _readurl(url, data=None, timeout=None, retries=0, sec_between=1,
              headers=None, headers_cb=None, ssl_details=None,
-             check_status=True, allow_redirects=True, exception_cb=None,
-             method='GET'):
+             check_status=True, allow_redirects=True, exception_cb=None):
     url = _cleanurl(url)
     req_args = {
         'url': url,
     }
     req_args.update(_get_ssl_args(url, ssl_details))
-    scheme = urlparse(url).scheme  # pylint: disable=E1101
     req_args['allow_redirects'] = allow_redirects
-    req_args['method'] = method
+    req_args['method'] = 'GET'
     if timeout is not None:
         req_args['timeout'] = max(float(timeout), 0)
     if data:

tests/unittests/test_datasource/test_openstack.py

@@ -117,20 +117,9 @@ def _register_uris(version, ec2_files, ec2_meta, os_files):
             return (200, headers, os_files.get(path))
         return match_ec2_url(uri, headers)
 
-    def head_request_callback(method, uri, headers):
-        uri = urlparse(uri)
-        path = uri.path.lstrip("/")
-        for key in os_files.keys():
-            if key.startswith(path):
-                return (200, headers, '')
-        return (404, headers, '')
-
     hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'),
                     body=get_request_callback)
 
-    hp.register_uri(hp.HEAD, re.compile(r'http://169.254.169.254/.*'),
-                    body=head_request_callback)
-
 
 class TestOpenStackDataSource(test_helpers.TestCase):
     VERSION = 'latest'