Add API client code from ceilometerclient

Contains code for making http requests as well as very basic foundation
classes for resources and their managers.

Also adds FakeAPI and FakeResponse classes to test utils.

tuskarclient/common/base.py

@@ -0,0 +1,142 @@
+# Copyright 2012 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Base utilities to build API operation managers and objects on top of.
+"""
+
+import copy
+
+
+# Python 2.4 compat
+try:
+    all
+except NameError:
+    def all(iterable):
+        return True not in (not x for x in iterable)
+
+
+def getid(obj):
+    """Abstracts the common pattern of allowing both an object or an
+    object's ID (UUID) as a parameter when dealing with relationships.
+    """
+    try:
+        return obj.id
+    except AttributeError:
+        return obj
+
+
+class Manager(object):
+    """Managers interact with a particular type of API
+    (samples, meters, alarms, etc.) and provide CRUD operations for them.
+    """
+    resource_class = None
+
+    def __init__(self, api):
+        self.api = api
+
+    def _create(self, url, body):
+        resp, body = self.api.json_request('POST', url, body=body)
+        if body:
+            return self.resource_class(self, body)
+
+    def _list(self, url, response_key=None, obj_class=None, body=None,
+              expect_single=False):
+        resp, body = self.api.json_request('GET', url)
+
+        if obj_class is None:
+            obj_class = self.resource_class
+
+        if response_key:
+            try:
+                data = body[response_key]
+            except KeyError:
+                return []
+        else:
+            data = body
+        if expect_single:
+            data = [data]
+        return [obj_class(self, res, loaded=True) for res in data if res]
+
+    def _update(self, url, body, response_key=None):
+        resp, body = self.api.json_request('PUT', url, body=body)
+        # PUT requests may not return a body
+        if body:
+            return self.resource_class(self, body)
+
+    def _delete(self, url):
+        self.api.raw_request('DELETE', url)
+
+
+class Resource(object):
+    """A resource represents a particular instance of an object (tenant, user,
+    etc). This is pretty much just a bag for attributes.
+
+    :param manager: Manager object
+    :param info: dictionary representing resource attributes
+    :param loaded: prevent lazy-loading if set to True
+    """
+    def __init__(self, manager, info, loaded=False):
+        self.manager = manager
+        self._info = info
+        self._add_details(info)
+        self._loaded = loaded
+
+    def _add_details(self, info):
+        for (k, v) in info.iteritems():
+            setattr(self, k, v)
+
+    def __getattr__(self, k):
+        if k not in self.__dict__:
+            # NOTE(bcwaldon): disallow lazy-loading if already loaded once
+            if not self.is_loaded():
+                self.get()
+                return self.__getattr__(k)
+
+            raise AttributeError(k)
+        else:
+            return self.__dict__[k]
+
+    def __repr__(self):
+        reprkeys = sorted(k for k in self.__dict__.keys() if k[0] != '_' and
+                          k != 'manager')
+        info = ", ".join("%s=%s" % (k, getattr(self, k)) for k in reprkeys)
+        return "<%s %s>" % (self.__class__.__name__, info)
+
+    def get(self):
+        # set_loaded() first ... so if we have to bail, we know we tried.
+        self.set_loaded(True)
+        if not hasattr(self.manager, 'get'):
+            return
+
+        new = self.manager.get(self.id)
+        if new:
+            self._add_details(new._info)
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return False
+        if hasattr(self, 'id') and hasattr(other, 'id'):
+            return self.id == other.id
+        return self._info == other._info
+
+    def is_loaded(self):
+        return self._loaded
+
+    def set_loaded(self, val):
+        self._loaded = val
+
+    def to_dict(self):
+        return copy.deepcopy(self._info)

tuskarclient/common/http.py

@@ -0,0 +1,284 @@
+# Copyright 2012 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+import httplib
+import logging
+import os
+import socket
+import StringIO
+import urlparse
+
+try:
+    import ssl
+except ImportError:
+    #TODO(bcwaldon): Handle this failure more gracefully
+    pass
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+# Python 2.5 compat fix
+if not hasattr(urlparse, 'parse_qsl'):
+    import cgi
+    urlparse.parse_qsl = cgi.parse_qsl
+
+
+from tuskarclient import exc
+
+
+LOG = logging.getLogger(__name__)
+USER_AGENT = 'python-tuskarclient'
+CHUNKSIZE = 1024 * 64  # 64kB
+
+
+class HTTPClient(object):
+
+    def __init__(self, endpoint, **kwargs):
+        self.endpoint = endpoint
+        self.auth_token = kwargs.get('token')
+        self.connection_params = self.get_connection_params(endpoint, **kwargs)
+
+    @staticmethod
+    def get_connection_params(endpoint, **kwargs):
+        parts = urlparse.urlparse(endpoint)
+
+        _args = (parts.hostname, parts.port, parts.path)
+        _kwargs = {'timeout': (float(kwargs.get('timeout'))
+                               if kwargs.get('timeout') else 600)}
+
+        if parts.scheme == 'https':
+            _class = VerifiedHTTPSConnection
+            _kwargs['ca_file'] = kwargs.get('ca_file', None)
+            _kwargs['cert_file'] = kwargs.get('cert_file', None)
+            _kwargs['key_file'] = kwargs.get('key_file', None)
+            _kwargs['insecure'] = kwargs.get('insecure', False)
+        elif parts.scheme == 'http':
+            _class = httplib.HTTPConnection
+        else:
+            msg = 'Unsupported scheme: %s' % parts.scheme
+            raise exc.InvalidEndpoint(msg)
+
+        return (_class, _args, _kwargs)
+
+    def get_connection(self):
+        _class = self.connection_params[0]
+        try:
+            return _class(*self.connection_params[1][0:2],
+                          **self.connection_params[2])
+        except httplib.InvalidURL:
+            raise exc.InvalidEndpoint()
+
+    def log_curl_request(self, method, url, kwargs):
+        curl = ['curl -i -X %s' % method]
+
+        for (key, value) in kwargs['headers'].items():
+            header = '-H \'%s: %s\'' % (key, value)
+            curl.append(header)
+
+        conn_params_fmt = [
+            ('key_file', '--key %s'),
+            ('cert_file', '--cert %s'),
+            ('ca_file', '--cacert %s'),
+        ]
+        for (key, fmt) in conn_params_fmt:
+            value = self.connection_params[2].get(key)
+            if value:
+                curl.append(fmt % value)
+
+        if self.connection_params[2].get('insecure'):
+            curl.append('-k')
+
+        if 'body' in kwargs:
+            curl.append('-d \'%s\'' % kwargs['body'])
+
+        curl.append('%s%s' % (self.endpoint, url))
+        LOG.debug(' '.join(curl))
+
+    @staticmethod
+    def log_http_response(resp, body=None):
+        status = (resp.version / 10.0, resp.status, resp.reason)
+        dump = ['\nHTTP/%.1f %s %s' % status]
+        dump.extend(['%s: %s' % (k, v) for k, v in resp.getheaders()])
+        dump.append('')
+        if body:
+            dump.extend([body, ''])
+        LOG.debug('\n'.join(dump))
+
+    def _make_connection_url(self, url):
+        (_class, _args, _kwargs) = self.connection_params
+        base_url = _args[2]
+        return '%s/%s' % (base_url.rstrip('/'), url.lstrip('/'))
+
+    def _http_request(self, url, method, **kwargs):
+        """Send an http request with the specified characteristics.
+
+        Wrapper around httplib.HTTP(S)Connection.request to handle tasks such
+        as setting headers and error handling.
+        """
+        # Copy the kwargs so we can reuse the original in case of redirects
+        kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {}))
+        kwargs['headers'].setdefault('User-Agent', USER_AGENT)
+        if self.auth_token:
+            kwargs['headers'].setdefault('X-Auth-Token', self.auth_token)
+
+        self.log_curl_request(method, url, kwargs)
+        conn = self.get_connection()
+
+        try:
+            conn_url = self._make_connection_url(url)
+            conn.request(method, conn_url, **kwargs)
+            resp = conn.getresponse()
+        except socket.gaierror as e:
+            message = "Error finding address for %(url)s: %(e)s" % locals()
+            raise exc.InvalidEndpoint(message=message)
+        except (socket.error, socket.timeout) as e:
+            endpoint = self.endpoint
+            message = "Error communicating with %(endpoint)s %(e)s" % locals()
+            raise exc.CommunicationError(message=message)
+
+        body_iter = ResponseBodyIterator(resp)
+
+        # Read body into string if it isn't obviously image data
+        if resp.getheader('content-type', None) != 'application/octet-stream':
+            body_str = ''.join([chunk for chunk in body_iter])
+            self.log_http_response(resp, body_str)
+            body_iter = StringIO.StringIO(body_str)
+        else:
+            self.log_http_response(resp)
+
+        if 400 <= resp.status < 600:
+            LOG.warn("Request returned failure status.")
+            raise exc.from_response(resp)
+        elif resp.status in (301, 302, 305):
+            # Redirected. Reissue the request to the new location.
+            return self._http_request(resp['location'], method, **kwargs)
+        elif resp.status == 300:
+            raise exc.from_response(resp)
+
+        return resp, body_iter
+
+    def json_request(self, method, url, **kwargs):
+        kwargs.setdefault('headers', {})
+        kwargs['headers'].setdefault('Content-Type', 'application/json')
+        kwargs['headers'].setdefault('Accept', 'application/json')
+
+        if 'body' in kwargs:
+            kwargs['body'] = json.dumps(kwargs['body'])
+
+        resp, body_iter = self._http_request(url, method, **kwargs)
+        content_type = resp.getheader('content-type', None)
+
+        if resp.status == 204 or resp.status == 205 or content_type is None:
+            return resp, list()
+
+        if 'application/json' in content_type:
+            body = ''.join([chunk for chunk in body_iter])
+            try:
+                body = json.loads(body)
+            except ValueError:
+                LOG.error('Could not decode response body as JSON')
+        else:
+            body = None
+
+        return resp, body
+
+    def raw_request(self, method, url, **kwargs):
+        kwargs.setdefault('headers', {})
+        kwargs['headers'].setdefault('Content-Type',
+                                     'application/octet-stream')
+        return self._http_request(url, method, **kwargs)
+
+
+class VerifiedHTTPSConnection(httplib.HTTPSConnection):
+    """httplib-compatibile connection using client-side SSL authentication
+
+    :see http://code.activestate.com/recipes/
+            577548-https-httplib-client-connection-with-certificate-v/
+    """
+
+    def __init__(self, host, port, key_file=None, cert_file=None,
+                 ca_file=None, timeout=None, insecure=False):
+        httplib.HTTPSConnection.__init__(self, host, port, key_file=key_file,
+                                         cert_file=cert_file)
+        self.key_file = key_file
+        self.cert_file = cert_file
+        if ca_file is not None:
+            self.ca_file = ca_file
+        else:
+            self.ca_file = self.get_system_ca_file()
+        self.timeout = timeout
+        self.insecure = insecure
+
+    def connect(self):
+        """Connect to a host on a given (SSL) port.
+        If ca_file is pointing somewhere, use it to check Server Certificate.
+
+        Redefined/copied and extended from httplib.py:1105 (Python 2.6.x).
+        This is needed to pass cert_reqs=ssl.CERT_REQUIRED as parameter to
+        ssl.wrap_socket(), which forces SSL to check server certificate against
+        our client certificate.
+        """
+        sock = socket.create_connection((self.host, self.port), self.timeout)
+
+        if self._tunnel_host:
+            self.sock = sock
+            self._tunnel()
+
+        if self.insecure is True:
+            kwargs = {'cert_reqs': ssl.CERT_NONE}
+        else:
+            kwargs = {'cert_reqs': ssl.CERT_REQUIRED, 'ca_certs': self.ca_file}
+
+        if self.cert_file:
+            kwargs['certfile'] = self.cert_file
+            if self.key_file:
+                kwargs['keyfile'] = self.key_file
+
+        self.sock = ssl.wrap_socket(sock, **kwargs)
+
+    @staticmethod
+    def get_system_ca_file():
+        """Return path to system default CA file."""
+        # Standard CA file locations for Debian/Ubuntu, RedHat/Fedora,
+        # Suse, FreeBSD/OpenBSD
+        ca_path = ['/etc/ssl/certs/ca-certificates.crt',
+                   '/etc/pki/tls/certs/ca-bundle.crt',
+                   '/etc/ssl/ca-bundle.pem',
+                   '/etc/ssl/cert.pem']
+        for ca in ca_path:
+            if os.path.exists(ca):
+                return ca
+        return None
+
+
+class ResponseBodyIterator(object):
+    """A class that acts as an iterator over an HTTP response."""
+
+    def __init__(self, resp):
+        self.resp = resp
+
+    def __iter__(self):
+        while True:
+            yield self.next()
+
+    def next(self):
+        chunk = self.resp.read(CHUNKSIZE)
+        if chunk:
+            return chunk
+        else:
+            raise StopIteration()

tuskarclient/tests/test_http.py

@@ -0,0 +1,48 @@
+# Copyright 2012 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from tuskarclient.tests import utils as tutils
+
+from tuskarclient.common import http
+
+
+fixtures = {}
+
+
+class HttpClientTest(tutils.TestCase):
+
+    def test_url_generation_trailing_slash_in_base(self):
+        client = http.HTTPClient('http://localhost/')
+        url = client._make_connection_url('/v1/resources')
+        print client.connection_params
+        self.assertEqual(url, '/v1/resources')
+
+    def test_url_generation_without_trailing_slash_in_base(self):
+        client = http.HTTPClient('http://localhost')
+        url = client._make_connection_url('/v1/resources')
+        print client.connection_params
+        self.assertEqual(url, '/v1/resources')
+
+    def test_url_generation_prefix_slash_in_path(self):
+        client = http.HTTPClient('http://localhost/')
+        url = client._make_connection_url('/v1/resources')
+        print client.connection_params
+        self.assertEqual(url, '/v1/resources')
+
+    def test_url_generation_without_prefix_slash_in_path(self):
+        client = http.HTTPClient('http://localhost')
+        url = client._make_connection_url('v1/resources')
+        print client.connection_params
+        self.assertEqual(url, '/v1/resources')

tuskarclient/tests/utils.py

@@ -10,10 +10,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import copy
 import fixtures
 import os
+import StringIO
 import testtools
 
+from tuskarclient.common import http
+
 
 class TestCase(testtools.TestCase):
 
@@ -27,3 +31,41 @@ class TestCase(testtools.TestCase):
                 os.environ.get('OS_STDERR_CAPTURE') == '1'):
             stderr = self.useFixture(fixtures.StringStream('stderr')).stream
             self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
+
+
+class FakeAPI(object):
+    def __init__(self, fixtures):
+        self.fixtures = fixtures
+        self.calls = []
+
+    def _request(self, method, url, headers=None, body=None):
+        call = (method, url, headers or {}, body)
+        self.calls.append(call)
+        return self.fixtures[url][method]
+
+    def raw_request(self, *args, **kwargs):
+        fixture = self._request(*args, **kwargs)
+        body_iter = http.ResponseBodyIterator(StringIO.StringIO(fixture[1]))
+        return FakeResponse(fixture[0]), body_iter
+
+    def json_request(self, *args, **kwargs):
+        fixture = self._request(*args, **kwargs)
+        return FakeResponse(fixture[0]), fixture[1]
+
+
+class FakeResponse(object):
+    def __init__(self, headers, body=None, version=None):
+        """:param headers: dict representing HTTP response headers
+        :param body: file-like object
+        """
+        self.headers = headers
+        self.body = body
+
+    def getheaders(self):
+        return copy.deepcopy(self.headers).items()
+
+    def getheader(self, key, default):
+        return self.headers.get(key, default)
+
+    def read(self, amt):
+        return self.body.read(amt)