Add abstract http method process

Add abstrace http method, such as POST, PUT, DELETE, GET.
Add some common execption and some unittest.

Change-Id: I65016ea2cba022823a80b044fc2e17801535a88b

.gitignore

@@ -17,6 +17,7 @@ develop-eggs
 .installed.cfg
 lib
 lib64
+.idea
 
 # Installer logs
 pip-log.txt
@@ -55,4 +56,4 @@ ChangeLog
 .*sw?
 
 # Files created by releasenotes build
-releasenotes/build
+releasenotes/build

requirements.txt

@@ -3,3 +3,5 @@
 # process, which may cause wedges in the gate later.
 
 pbr>=1.8 # Apache-2.0
+oslo.utils>=3.18.0 # Apache-2.0
+oslo.i18n>=2.1.0 # Apache-2.0

test-requirements.txt

@@ -12,6 +12,8 @@ oslotest>=1.10.0 # Apache-2.0
 testrepository>=0.0.18  # Apache-2.0/BSD
 testscenarios>=0.4  # Apache-2.0/BSD
 testtools>=1.4.0 # MIT
+oslo.utils>=3.18.0 # Apache-2.0
+oslo.i18n>=2.1.0 # Apache-2.0
 
 # releasenotes
 reno>=1.8.0 # Apache-2.0

valenceclient/common/apiclient/base.py

@@ -0,0 +1,443 @@
+# Copyright 2016 99cloud, Inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+
+from oslo_utils import strutils
+from six.moves import http_client
+from six.moves.urllib import parse
+
+from valenceclient.common.apiclient import exceptions
+from valenceclient.common.i18n import _
+
+
+def getid(obj):
+    """Return id if argument is a Resource"""
+
+    try:
+        if obj.uuid:
+            return obj.uuid
+    except AttributeError:
+        pass
+
+    try:
+        if obj.id:
+            return obj.id
+    except AttributeError:
+        return obj
+
+
+# TODO(aababilov): call run_hooks() in HookableMixin's child classes
+class HookableMixin(object):
+    """Mixin so classes can register and run hooks."""
+
+    _hook_map = {}
+
+    @classmethod
+    def add_hook(cls, hook_type, hook_func):
+        """Add a new hook of specified type.
+
+        :param cls: class that registers hooks
+        :param hook_type: hook type
+        :param hook_func: hook function
+        """
+
+        if hook_type not in cls._hook_map:
+            cls._hook_map[hook_type] = []
+        cls._hook_map[hook_type].append(hook_func)
+
+    @classmethod
+    def run_hooks(cls, hook_type, *args, **kwargs):
+        """Run all hooks of specified type.
+
+        :param cls: class that registers hooks
+        :param hook_type: hook type
+        :param args: args to be passed to every hook function
+        :param kwargs: kwargs to be passwd to every hook function
+        """
+
+        hook_funcs = cls._hook_map.get(hook_type) or []
+        for hook_func in hook_funcs:
+            hook_func(*args, **kwargs)
+
+
+class BaseManager(HookableMixin):
+    """Basic manager type providing common operations.
+
+    Managers interact with a praticular type of API and provide CRUD
+    operations for them
+    """
+
+    resource_class = None
+
+    def __init__(self, client):
+        """Initializes BaseManager with `client`.
+
+        :param client: instance of BaseClient descendant for HTTP
+            requests
+        """
+        super(BaseManager, self).__init__()
+        self.client = client
+
+    def _list(self, url, response_key=None, obj_class=None, json=None):
+        """List the collection.
+
+        :param url: a partial URL, e.g., '/servers'
+        :param response_key: the key to be looked up in response dictionary,
+            e.g., 'servers'. If response_key is None - all response body
+            will be used.
+        :param obj_class: class for constructing the returned objects
+            (self.resource_class will be used by default)
+        :param json: data that will be encoded as JSON and passed in POST
+            request (GET will be sent by default)
+        """
+
+        if json:
+            body = self.client.post(url, json=json).json()
+        else:
+            body = self.client.post(url).json()
+
+        if obj_class is None:
+            obj_class = self.resource_class
+
+        data = body[response_key] if response_key is not None else body
+        # NOTE(ja): keystone returns values as list as {'values': [ ... ]}
+        #           unlike other services which just return the list...
+        try:
+            data = data['values']
+        except (KeyError, TypeError):
+            pass
+
+        return [obj_class(self, res, loaded=True) for res in data if res]
+
+    def _get(self, url, response_key=None):
+        """Get an object from collection.
+
+        :param url: a partial URL, e.g., '/servers'
+        :param response_key: the key to be looked up in response dictionary,
+            e.g., 'servers'. If response_key is None - all response body
+            will be used.
+        """
+
+        body = self.client.get(url).json()
+        data = body[response_key] if response_key is not None else body
+        return self.resource_class(self, data, loaded=True)
+
+    def _head(self, url):
+        """Get head information
+
+        :param url: a partial URL, e.g., '/servers'
+        :return:
+        """
+
+        resp = self.client.head(url)
+        return resp.status_code == http_client.NO_CONTENT
+
+    def _post(self, url, json, response_key=None, return_raw=False):
+        """Create an object.
+
+        :param url: a partial URL, e.g., '/servers'
+        :param json: data that will be encoded as JSON and passed in POST
+            request (GET will be sent by default)
+        :param response_key:the key to be looked up in response dictionary,
+            e.g., 'server'. If response_key is None - all response body
+            will be used.
+        :param return_raw:flag to force returning raw JSON instead of
+            Python object of self.resource_class
+        :return:
+        """
+
+        body = self.client.post(url, json=json).json()
+        data = body[response_key] if response_key is not None else body
+        if return_raw:
+            return data
+        return self.resource_class(self, data)
+
+    def _put(self, url, json=None, response_key=None):
+        """Update an object with PUT method
+
+        :param url: a partial URL, e.g., '/servers'
+        :param json: data that will be encoded as JSON and passed in POST
+            request (GET will be sent by default)
+        :param response_key: the key to be looked up in response dictionary,
+            e.g., 'server'. If response_key is None - all response body
+            will be used.
+        :return:
+        """
+
+        resp = self.client.put(url, json=json)
+        if resp.content:
+            body = resp.json()
+            if response_key is not None:
+                return self.resource_class(self, body[response_key])
+            else:
+                return self.resource_class(self, body)
+
+    def _delete(self, url):
+        """Delete an object
+
+        :param url:a partial URL, e.g., '/servers'
+        :return:
+        """
+
+        return self.client.delete(url)
+
+
+class CrudManager(BaseManager):
+    """Base manager class for manipulating entities.
+
+    Children of this class are expected to define a `collection_key` and `key`.
+    - `collection_key`: Usually a plural noun by convention (e.g. `entities`);
+      used to refer collections in both URL's (e.g.  `/v3/entities`) and JSON
+      objects containing a list of member resources (e.g. `{'entities': [{},
+      {}, {}]}`).
+    - `key`: Usually a singular noun by convention (e.g. `entity`); used to
+      refer to an individual member of the collection.
+    """
+
+    collection_key = None
+    key = None
+
+    def build_url(self, base_url=None, **kwargs):
+        """Builds a resource url for the given kwargs.
+
+        Given an example collection where `collection_key = 'entities'` and
+        `key = 'entitied'`, the following URL's could be generated.
+
+        By default, the URL will represent a collection of entities, e.g.::
+
+            /entities
+
+        If kwargs
+        :param base_url:
+        :param kwargs:
+        :return:
+        """
+
+        url = base_url if base_url is not None else ''
+        url += '/%s' % self.collection_key
+
+        # do we have a specific entity?
+        entity_id = kwargs.get('%s_id' % self.key)
+        if entity_id is not None:
+            url += '/%s' % entity_id
+
+        return url
+
+    def _filter_kwargs(self, kwargs):
+        """Drop null values and handle ids."""
+
+        for key, ref in kwargs.copy().items():
+            if ref in None:
+                kwargs.pop(key)
+            else:
+                if isinstance(ref, Resource):
+                    kwargs.pop(key)
+                    kwargs['%s_id' % key] = getid(ref)
+
+    def create(self, **kwargs):
+        kwargs = self._filter_kwargs(kwargs)
+        return self._post(self.build_url(**kwargs),
+                          {self.key: kwargs}, self.key)
+
+    def get(self, **kwargs):
+        kwargs = self._filter_kwargs(kwargs)
+        return self._get(self.build_url(**kwargs), self.key)
+
+    def head(self, **kwargs):
+        kwargs = self._filter_kwargs(kwargs)
+        return self._head(self.build_url(**kwargs))
+
+    def list(self, base_url=None, **kwargs):
+        """List the collection.
+
+        :param base_url: if provided, the generated URL will be appended to it
+        """
+
+        kwargs = self._filter_kwargs(kwargs)
+        return self._list('%(base_url)s%(query)s' % {
+            'base_url': self.build_url(base_url=base_url, **kwargs),
+            'query': '?%s' % parse.urlencode(kwargs) if kwargs else ''
+            }, self.collection_key)
+
+    def put(self, base_url=None, **kwargs):
+        """Update an element.
+
+        :param base_url: if provided, the generated URL will be appended to it
+        """
+
+        kwargs = self._filter_kwargs(kwargs)
+        return self._put(self.build_url(base_url=base_url), **kwargs)
+
+    def update(self, **kwargs):
+        kwargs = self._filter_kwargs(kwargs)
+        params = kwargs.copy()
+        params.pop('%s_id' % self.key)
+
+        return self._patch(self.build_url(**kwargs), {self.key: params},
+                           self.key)
+
+    def delete(self, **kwargs):
+        kwargs = self._filter_kwargs(kwargs)
+
+        return self._delete(self.build_url(**kwargs))
+
+    def find(self, base_url=None, **kwargs):
+        """Find a single item with attributes matching ``**kwargs``.
+
+        :param base_url: if provided, the generated URL will be appended to it
+        """
+
+        kwargs = self._filter_kwargs(kwargs)
+
+        rl = self._list('%(base_url)s%(query)s' % {
+            'base_url': self.build_url(base_url=base_url, **kwargs),
+            'query': '?%s' % parse.urlencode(kwargs) if kwargs else ''
+            }, self.collection_key)
+
+        num = len(rl)
+        if num == 0:
+            msg = _("No %(name)s matching %(args)s.") % {
+                'name': self.resource_class.__name__,
+                'args': kwargs
+            }
+            raise exceptions.NotFound(msg)
+        elif num > 1:
+            msg = _("More than one result matching %(args)s.") % {
+                'args': kwargs}
+            raise exceptions.NoUniqueMatch(msg)
+        else:
+            return rl[0]
+
+
+class Extension(HookableMixin):
+    """Extension descriptor."""
+
+    SUPPORTED_HOOKS = ('__pre_parse_args__', '__post_parse_args__')
+    manager_class = None
+
+    def __init__(self, name, module):
+        super(Extension, self).__init__()
+        self.name = name
+        self.module = module
+        self._parse_extension_module()
+
+    def _parse_extension_module(self):
+        self.manager_class = None
+        for attr_name, attr_value in self.module.__dict__.items():
+            if attr_name in self.SUPPORTED_HOOKS:
+                self.add_hook(attr_name, attr_value)
+            else:
+                try:
+                    if issubclass(attr_value, BaseManager):
+                        self.manager_class = attr_value
+                except TypeError:
+                    pass
+
+    def __repr__(self):
+        return "<Extension '%s'>" % self.name
+
+
+class Resource(object):
+    """Base class for OpenStack resources (tenant, user, etc.).
+
+    This is pretty much just a bag for attributes.
+    """
+
+    HUMAN_ID = False
+    NAME_ATTTR = 'name'
+
+    def __init__(self, manager, info, loaded=False):
+        """Populate and bind to a manager.
+
+        :param manager: BaseManager object
+        :param info: dictionary representing resource attributes
+        :param loaded: prevent lazy-loading if set to True
+        """
+
+        self.manager = manager
+        self._info = info
+        self._loaded = loaded
+        self._add_details(info)
+
+    def __repr__(self):
+        reprkeys = sorted(k for k in self.__dict__.keys()
+                          if k[0] != '_' and k != 'manager')
+        info = ", ".join("%s=%s" % (k, getattr(self, k)) for k in reprkeys)
+        return "<%s %s>" % (self.__class__.__name__, info)
+
+    @property
+    def human_id(self):
+        """Human-readable ID which can be used for bash completion."""
+
+        if self.HUMAN_ID:
+            name = getattr(self, self.NAME_ATTTR, None)
+            if name is not None:
+                return strutils.to_slug(name)
+        return None
+
+    def _add_details(self, info):
+        for(k, v) in info.items():
+            try:
+                setattr(self, k, v)
+                self._info[k] = v
+            except AttributeError:
+                # In this case we already defined the attribute on the class
+                pass
+
+    def __getattr__(self, k):
+        if k not in self.__dict__:
+            # NOTE(bcwaldon): disallow lazy-loading if already loaded once
+            if not self.is_loaded():
+                self.get()
+                return self.__getattr__(k)
+
+            raise AttributeError
+        else:
+            return self.__dict__[k]
+
+    def get(self):
+        """Support for lazy loading details.
+
+        Some clients, such as novaclient have the option to lazy load the
+        details, details which can be loaded with this function.
+        """
+
+        # set_loaded() first ... so if we have to bail, we know we tried.
+        self.set_loaded(True)
+        if not hasattr(self.manager, 'get'):
+            return
+        new = self.manager.get(self.id)
+        if new:
+            self._add_details(new._info)
+            self._add_details(
+                {'x_request_id': self.manager.client.last_request_id})
+
+    def __eq__(self, other):
+        if not isinstance(other, Resource):
+            return NotImplemented
+        # two resources of different types are not equal
+        if not isinstance(other, self.__class__):
+            return False
+        return self._info == other._info
+
+    def is_loaded(self):
+        return self._loaded
+
+    def set_loaded(self, val):
+        self._loaded = val
+
+    def to_dict(self):
+        return copy.deepcopy(self._info)

valenceclient/common/apiclient/exceptions.py

@@ -0,0 +1,139 @@
+# Copyright 2016 99cloud, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Exception definitions
+"""
+
+import inspect
+import sys
+
+import six
+from six.moves import http_client
+
+from valenceclient.common.i18n import _
+
+
+class ClientException(Exception):
+    """The base exception class for all exceptions this library raises"""
+
+    message = _("ClientException")
+
+
+class HttpError(ClientException):
+    """The base exception class of all HTTP exceptions"""
+
+    http_status = 0
+    message = _("HTTP Error")
+
+    def __init__(self, message=None, details=None, response=None,
+                 request_id=None, url=None, method=None,
+                 http_status=None):
+        self.message = message or self.message
+        self.details = details
+        self.response = response
+        self.request_id = request_id
+        self.url = url
+        self.method = method
+        self.http_status = http_status
+
+        formatted_string = "%s (HTTP %s)" % (self.message, self.http_status)
+        if request_id:
+            formatted_string += " (Request-ID: %s)" % (self.request_id)
+
+        super(HttpError, self).__init__(formatted_string)
+
+
+class HTTPClientError(HttpError):
+    """Client-side HTTP error.
+
+    Exception for cases in which the client seems to have erred.
+    """
+
+    message = _("HTTP Client Error")
+
+
+class HttpServerError(HttpError):
+    """Server-side HTTP error.
+
+    Exception for cases in which the server is aware that it has
+    erred or is incapable of performing the request.
+    """
+
+    message = _("HTTP Server Error")
+
+
+# _code_map cotains all the classes that have http_status attribute
+_code_map = dict((getattr(obj, 'http_status', None), obj)
+                 for name, obj in vars(sys.modules[__name__]).items()
+                 if inspect.isclass(obj) and
+                 getattr(obj, 'http_status', False))
+
+
+def from_response(response, method, url):
+    """Returns an instance of :class:`HttpError` or subclass based on response.
+
+    :param response: instance of `requests.Response` class
+    :param method: HTTP method used for request
+    :param url: URL used for request
+    """
+
+    req_id = response.headers.get('X-openstack-request-id')
+    # NOTE(hdd) true for older versions of nova and cinder
+    if not req_id:
+        req_id = response.headers.get('X-compute-request-id')
+
+    kwages = {
+        'http_status': response.status_code,
+        'response': response,
+        'method': method,
+        'url': url,
+        'request_id': req_id
+    }
+
+    if 'retry_after' in response.headers:
+        kwages['retry_after'] = response.headers['retry_after']
+
+    content_type = response.headers.get('Content-Type', "")
+    if content_type.startswith('application/json'):
+        try:
+            body = response.json()
+        except ValueError:
+            pass
+        else:
+            if isinstance(body, dict):
+                error = body.get(list(body)[0])
+                if isinstance(error, dict):
+                    kwages['message'] = (error.get('message') or
+                                         error.get('faultstring'))
+                    kwages['details'] = (error.get('details') or
+                                         six.text_type(body))
+    elif content_type.startswith("text/"):
+        kwages['details'] = getattr(response, 'text', '')
+
+    try:
+        cls = _code_map[response.status_code]
+    except KeyError:
+        # 5XX status codes are server errors
+        if response.status_code >= http_client.INTERNAL_SERVER_ERROR:
+            cls = HttpServerError
+        # 4XX status codes are client request errors
+        elif (http_client.BAD_REQUEST <= response.status_code <
+              http_client.INTERNAL_SERVER_ERROR):
+            cls = HTTPClientError
+        else:
+            cls = HttpError
+
+    return cls

valenceclient/common/i18n.py

@@ -0,0 +1,31 @@
+# Copyright 2016 99cloud, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import oslo_i18n
+
+_translators = oslo_i18n.TranslatorFactory(domain='valenceclient')
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = _translators.log_info
+_LW = _translators.log_warning
+_LE = _translators.log_error
+_LC = _translators.log_critical

valenceclient/tests/base.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2010-2011 OpenStack Foundation
-# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from oslotest import base
-
-
-class TestCase(base.BaseTestCase):
-
-    """Test case base class for all unit tests."""

valenceclient/tests/test_valenceclient.py

@@ -1,28 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""
-test_valenceclient
-----------------------------------
-
-Tests for `valenceclient` module.
-"""
-
-from valenceclient.tests import base
-
-
-class TestValenceclient(base.TestCase):
-
-    def test_something(self):
-        pass

valenceclient/tests/unit/common/apiclient/test_base.py

@@ -0,0 +1,68 @@
+# Copyright 2016 99cloud, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from oslotest import base as test_base
+from valenceclient.common.apiclient import base
+
+
+class HumanResource(base.Resource):
+    HUMAN_ID = True
+
+
+class TestBase(test_base.BaseTestCase):
+
+    def test_resource_repr(self):
+        r = base.Resource(None, dict(foo="bar", baz="spam"))
+        self.assertEqual("<Resource baz=spam, foo=bar>", repr(r))
+
+    def test_getid(self):
+        self.assertEqual(4, base.getid(4))
+
+        class TmpObject(object):
+            id = 4
+        self.assertEqual(4, base.getid(TmpObject))
+
+    def test_human_id(self):
+        r = base.Resource(None, {"name": "1"})
+        self.assertIsNone(r.human_id)
+        r = HumanResource(None, {"name": "1"})
+        self.assertEqual("1", r.human_id)
+        r = HumanResource(None, {"name": None})
+        self.assertIsNone(r.human_id)
+
+    def test_two_resources_with_same_id_are_not_equal(self):
+        # Two resources with same ID: never equal if their info is not equal
+        r1 = base.Resource(None, {'id': 1, 'name': 'hi'})
+        r2 = base.Resource(None, {'id': 1, 'name': 'hello'})
+        self.assertNotEqual(r1, r2)
+
+    def test_two_resources_with_same_id_and_info_are_equal(self):
+        # Two resources with same ID: equal if their info is equal
+        r1 = base.Resource(None, {'id': 1, 'name': 'hello'})
+        r2 = base.Resource(None, {'id': 1, 'name': 'hello'})
+        self.assertEqual(r1, r2)
+
+    def test_two_resources_with_diff_type_are_not_equal(self):
+        # Two resoruces of different types: never equal
+        r1 = base.Resource(None, {'id': 1})
+        r2 = HumanResource(None, {'id': 1})
+        self.assertNotEqual(r1, r2)
+
+    def test_two_resources_with_no_id_are_equal(self):
+        # Two resources with no ID: equal if their info is equal
+        r1 = base.Resource(None, {'name': 'joe', 'age': 12})
+        r2 = base.Resource(None, {'name': 'joe', 'age': 12})
+        self.assertEqual(r1, r2)