Add basic client for HVN REST API

.pylintrc

@@ -99,7 +99,7 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme
 [MISCELLANEOUS]
 
 # List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
+notes=FIXME,XXX
 
 
 [FORMAT]

hnv_client/common/constant.py

@@ -13,3 +13,22 @@
 #    under the License.
 
 """Shared constants across the bcbio-nextgen-vm project."""
+
+GET = "GET"
+POST = "POST"
+PUT = "PUT"
+PATCH = "PATCH"
+DELETE = "DELETE"
+
+DELETING = "Deleting"
+FAILED = "Failed"
+SUCCEEDED = "Succeeded"
+UPDATING = "Updating"
+
+ABSOLUTE = "absolute"
+WEIGHT = "weight"
+
+VIRTUAL_APPLIANCE = "VirtualAppliance"
+VNET_LOCAL = "VnetLocal"
+VIRTUAL_NETWORK_GATEWAY = "VirtualNetworkGateway"
+INTERNET = "Internet"

hnv_client/common/exception.py

@@ -58,15 +58,41 @@ class DataProcessingError(HNVException):
     template = "The provided information is incomplete or invalid."
 
 
-class NotFound(HNVException):
+class ServiceException(HNVException):
+
+    """Base exception for all the API interaction related errors."""
+
+    template = "Something went wrong."
+
+
+class TimeOut(ServiceException):
+
+    """The request timed out."""
+
+    template = "The request timed out."
+
+
+class NotFound(ServiceException):
 
     """The required object is not available in container."""
 
     template = "The %(object)r was not found in %(container)s."
 
 
-class NotSupported(HNVException):
+class CertificateVerifyFailed(ServiceException):
+
+    """The received certificate is not valid.
+
+    In order to avoid the current exception the validation of the SSL
+    certificate should be disabled for the metadata provider. In order
+    to do that the `https_allow_insecure` config option should be set.
+    """
+
+    template = "The received certificate is not valid."
+
+
+class NotSupported(ServiceException):
 
     """The functionality required is not available in the current context."""
 
-    template = "%(feature)s is not available in %(context)s."
+    template = "%(feature)s is not available for %(context)s."

hnv_client/common/utils.py

@@ -0,0 +1,202 @@
+# Copyright 2017 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Utilities used across the project."""
+
+import json
+import sys
+import time
+
+from oslo_log import log as logging
+import requests
+import requests_ntlm
+import six
+
+from hnv_client.common import constant
+from hnv_client.common import exception
+from hnv_client import config as hnv_config
+
+LOG = logging.getLogger(__name__)
+CONFIG = hnv_config.CONFIG
+
+
+class _HNVClient(object):
+
+    """Minimalistic client for the Network Controller REST API.
+
+    :param url:             The base URL where the agent looks for
+                                Network Controller API.
+    :param username:        The username required for connecting to the
+                            Network Controller API.
+    :param password:        The password required for connecting to the
+                            Network Controller API.
+    :param allow_insecure:  Whether to disable the validation of
+                            HTTPS certificates.
+    :param ca_bundle:       The path to a CA_BUNDLE file or directory
+                            with certificates of trusted CAs.
+    """
+
+    def __init__(self, url, username=None, password=None, allow_insecure=False,
+                 ca_bundle=None):
+        self._base_url = url
+        self._credentials = (username, password)
+        self._https_allow_insecure = allow_insecure
+        self._https_ca_bundle = ca_bundle
+        self._http_session = None
+
+    @property
+    def _session(self):
+        """The current session used by the client.
+
+        The Session object allows you to persist certain parameters across
+        requests. It also persists cookies across all requests made from
+        the Session instance, and will use urllib3's connection pooling.
+        So if you're making several requests to the same host, the underlying
+        TCP connection will be reused, which can result in a significant
+        performance increase.
+        """
+        if self._http_session is None:
+            self._http_session = requests.Session()
+            self._http_session.headers.update(self._get_headers())
+            self._http_session.verify = self._verify_https_request()
+
+            if all(self._credentials):
+                username, password = self._credentials
+                self._http_session.auth = requests_ntlm.HttpNtlmAuth(
+                    username=username, password=password)
+
+        return self._http_session
+
+    @staticmethod
+    def _get_headers():
+        """Prepare the HTTP headers for the current request."""
+
+        # TODO(alexcoman): Add the x-ms-client-ip-address header in order
+        # to improve the Network Controller requests logging.
+        return {
+            "Accept": "application/json",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json; charset=UTF-8",
+        }
+
+    def _verify_https_request(self):
+        """Whether to disable the validation of HTTPS certificates.
+
+        .. notes::
+            When `https_allow_insecure` option is `True` the SSL certificate
+            validation for the connection with the Network Controller API will
+            be disabled (please don't use it if you don't know the
+            implications of this behaviour).
+        """
+        if self._https_ca_bundle:
+            return self._https_ca_bundle
+        else:
+            return not self._https_allow_insecure
+
+    def _http_request(self, resource, method=constant.GET, body=None):
+        url = requests.compat.urljoin(self._base_url, resource)
+        headers = self._get_headers()
+        if method in (constant.PUT, constant.PATCH):
+            etag = (body or {}).get("etag", None)
+            if etag is not None:
+                headers["If-Match"] = etag
+
+        attemts = 0
+        while True:
+            try:
+                response = self._session.request(
+                    method=method, url=url, headers=headers,
+                    data=json.dumps(body) if body else None,
+                    timeout=CONFIG.HNV.http_request_timeout
+                )
+                break
+            except requests.ConnectionError as exc:
+                attemts += 1
+                self._http_session = None
+                LOG.debug("Request failed: %s", exc)
+                if attemts > CONFIG.HNV.retry_count:
+                    if isinstance(exc, requests.exceptions.SSLError):
+                        raise exception.CertificateVerifyFailed(
+                            "HTTPS certificate validation failed.")
+                    raise
+                time.sleep(CONFIG.HNV.retry_interval)
+
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as exc:
+            status_code = exc.response.status_code
+            content = exc.response.text
+            LOG.debug("HTTP Error %(status_code)r: %(details)r",
+                      {"status_code": status_code, "details": content})
+
+            if status_code == 400:
+                raise exception.ServiceException(
+                    ("HNV Client failed to communicate with the API. "
+                     "Please open an issue with the following information: "
+                     "%(resource)r: %(details)r"),
+                    resource=resource, details=content
+                )
+            if status_code == 404:
+                raise exception.NotFound(
+                    "Resource %(resource)r was not found.", resource=resource)
+            raise
+
+        return response
+
+    def get_resource(self, path):
+        """Getting the required information from the API."""
+        response = self._http_request(path)
+        try:
+            return response.json()
+        except ValueError:
+            raise exception.ServiceException("Invalid service response.")
+
+    def update_resource(self, path, data):
+        """Update the required resource."""
+        response = self._http_request(path, method="PUT", body=data)
+        try:
+            return response.json()
+        except ValueError:
+            raise exception.ServiceException("Invalid service response.")
+
+    def remove_resource(self, path):
+        """Delete the received resource."""
+        return self._http_request(path, method="DELETE")
+
+
+# pylint: disable=dangerous-default-value
+def run_once(function, state={}, errors={}):
+    """A memoization decorator, whose purpose is to cache calls."""
+    @six.wraps(function)
+    def _wrapper(*args, **kwargs):
+        if function in errors:
+            # Deliberate use of LBYL.
+            six.reraise(*errors[function])
+
+        try:
+            return state[function]
+        except KeyError:
+            try:
+                state[function] = result = function(*args, **kwargs)
+                return result
+            except Exception:
+                errors[function] = sys.exc_info()
+                raise
+    return _wrapper
+
+
+@run_once
+def get_client(url, username, password, allow_insecure, ca_bundle):
+    """Create a new client for the HNV REST API."""
+    return _HNVClient(url, username, password, allow_insecure, ca_bundle)

hnv_client/config/hnv.py

@@ -47,10 +47,22 @@ class HVNOptions(config_base.Options):
                 "https_ca_bundle", default=None,
                 help=("The path to a CA_BUNDLE file or directory with "
                       "certificates of trusted CAs.")),
+            cfg.IntOpt(
+                "retry_count", default=5,
+                help="Max. number of attempts for fetching metadata in "
+                     "case of transient errors"),
             cfg.FloatOpt(
                 "retry_interval", default=1,
                 help=("Interval between attempts in case of transient errors, "
                       "expressed in seconds")),
+            cfg.IntOpt(
+                "http_request_timeout", default=None,
+                help=("Number of seconds until network requests stop waiting "
+                      "for a response")),
+            cfg.StrOpt(
+                "logical_network", default=None,
+                help=("Logical network to use as a medium for tenant network "
+                      "traffic.")),
         ]
 
     def register(self):

hnv_client/tests/common/test_utils.py

@@ -0,0 +1,253 @@
+# Copyright 2017 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# pylint: disable=protected-access, missing-docstring
+
+import unittest
+try:
+    import unittest.mock as mock
+except ImportError:
+    import mock
+
+import requests
+
+from hnv_client.common import constant
+from hnv_client.common import exception
+from hnv_client.common import utils as hnv_utils
+from hnv_client import config as hnv_config
+from hnv_client.tests import utils as test_utils
+
+CONFIG = hnv_config.CONFIG
+
+
+class TestHNVClient(unittest.TestCase):
+
+    @staticmethod
+    def _get_client(url=mock.sentinel.url, username=mock.sentinel.username,
+                    password=mock.sentinel.password,
+                    allow_insecure=mock.sentinel.insecure,
+                    ca_bundle=mock.sentinel.ca_bundle):
+        return hnv_utils._HNVClient(url, username, password, allow_insecure,
+                                    ca_bundle)
+
+    @mock.patch("hnv_client.common.utils._HNVClient._get_headers")
+    @mock.patch("hnv_client.common.utils._HNVClient._verify_https_request")
+    @mock.patch("requests_ntlm.HttpNtlmAuth")
+    @mock.patch("requests.Session")
+    def test_session(self, mock_get_session, mock_auth, mock_verify,
+                     mock_headers):
+        mock_session = mock.Mock()
+        mock_session.headers = {}
+        mock_get_session.return_value = mock_session
+        mock_verify.return_value = mock.sentinel.verify
+        mock_auth.return_value = mock.sentinel.auth
+        mock_headers.return_value = {"X-HNV-Test": 1}
+
+        client = self._get_client()
+        session = client._session
+
+        self.assertIs(session, mock_session)
+        self.assertIs(mock_session.verify, mock.sentinel.verify)
+        self.assertIs(mock_session.auth, mock.sentinel.auth)
+        self.assertEqual(mock_session.headers.get("X-HNV-Test"), 1)
+        mock_auth.assert_called_once_with(username=mock.sentinel.username,
+                                          password=mock.sentinel.password)
+
+    def test_verify_https_request(self):
+        ca_bundle_client = self._get_client(allow_insecure=None)
+        insecure_client = self._get_client(ca_bundle=None)
+
+        self.assertIs(ca_bundle_client._verify_https_request(),
+                      mock.sentinel.ca_bundle)
+        self.assertFalse(insecure_client._verify_https_request())
+
+    @mock.patch("time.sleep")
+    @mock.patch("json.dumps")
+    @mock.patch("requests.compat.urljoin")
+    @mock.patch("hnv_client.common.utils._HNVClient._session")
+    @mock.patch("hnv_client.common.utils._HNVClient._get_headers")
+    def _test_http_request(self, mock_headers, mock_session, mock_join,
+                           mock_dump, mock_sleep,
+                           method, body, response, status_code):
+        output = []
+        headers = mock_headers.return_value = {}
+        mock_join.return_value = mock.sentinel.url
+        mock_dump.return_value = mock.sentinel.content
+
+        session_request = mock_session.request = mock.MagicMock()
+        session_request.side_effect = response
+
+        expected_response = response[-1]
+
+        status_check = expected_response.raise_for_status = mock.MagicMock()
+        if status_code != 200:
+            exc_response = mock.MagicMock()
+            exc_response.status_code = status_code
+            exc_response.text = "Expected Error"
+            status_check.side_effect = requests.HTTPError(
+                response=exc_response)
+            output.append("HTTP Error %(status_code)r: 'Expected Error'" %
+                          {"status_code": status_code})
+
+        client = self._get_client()
+        with test_utils.LogSnatcher("hnv_client.common.utils") as logging:
+            if isinstance(expected_response, requests.exceptions.SSLError):
+                self.assertRaises(exception.CertificateVerifyFailed,
+                                  client._http_request,
+                                  mock.sentinel.resource, method, body)
+                return
+            elif isinstance(expected_response, requests.ConnectionError):
+                self.assertRaises(requests.ConnectionError,
+                                  client._http_request,
+                                  mock.sentinel.resource, method, body)
+                return
+            elif status_code == 400:
+                self.assertRaises(exception.ServiceException,
+                                  client._http_request,
+                                  mock.sentinel.resource, method, body)
+            elif status_code == 404:
+                self.assertRaises(exception.NotFound,
+                                  client._http_request,
+                                  mock.sentinel.resource, method, body)
+            elif status_code != 200:
+                self.assertRaises(requests.HTTPError,
+                                  client._http_request,
+                                  mock.sentinel.resource, method, body)
+            else:
+                client_response = client._http_request(mock.sentinel.resource,
+                                                       method, body)
+
+        mock_join.assert_called_once_with(mock.sentinel.url,
+                                          mock.sentinel.resource)
+        mock_headers.assert_called_once_with()
+        if not method == constant.GET:
+            etag = (body or {}).get("etag", None)
+            self.assertEqual(headers["If-Match"], etag)
+
+        if len(response) == 1:
+            session_request.assert_called_once_with(
+                method=method, url=mock.sentinel.url, headers=headers,
+                data=mock.sentinel.content if body else None,
+                timeout=CONFIG.HNV.http_request_timeout
+            )
+        elif len(response) > 1:
+            # Note(alexcoman): The first response is an exception
+            output.append("Request failed: ")
+
+        self.assertEqual(logging.output, output)
+        if status_code == 200:
+            self.assertIs(client_response, expected_response)
+
+    def test_http_request_get(self):
+        response = [mock.MagicMock()]
+        self._test_http_request(method=constant.GET,
+                                body=mock.sentinel.body,
+                                response=response,
+                                status_code=200)
+
+    def test_http_request_put(self):
+        response = [mock.MagicMock()]
+        self._test_http_request(method=constant.PUT,
+                                body={"etag": mock.sentinel.etag},
+                                response=response,
+                                status_code=200)
+
+    def test_http_request_with_connection_error(self):
+        response = [requests.ConnectionError(), mock.MagicMock()]
+        with test_utils.ConfigPatcher('retry_count', 1, "HNV"):
+            self._test_http_request(method=constant.GET,
+                                    body=mock.sentinel.body,
+                                    response=response,
+                                    status_code=200)
+
+    def test_http_request_connection_error(self):
+        response = [requests.ConnectionError(), requests.ConnectionError()]
+        with test_utils.ConfigPatcher('retry_count', 1, "HNV"):
+            self._test_http_request(method=constant.GET,
+                                    body=mock.sentinel.body,
+                                    response=response,
+                                    status_code=200)
+
+    def test_http_request_ssl_error(self):
+        response = [requests.exceptions.SSLError(),
+                    requests.exceptions.SSLError()]
+        with test_utils.ConfigPatcher('retry_count', 1, "HNV"):
+            self._test_http_request(method=constant.GET,
+                                    body=mock.sentinel.body,
+                                    response=response,
+                                    status_code=200)
+
+    def test_http_request_not_found(self):
+        response = [mock.MagicMock()]
+        self._test_http_request(method=constant.GET,
+                                body=mock.sentinel.body,
+                                response=response,
+                                status_code=404)
+
+    def test_http_request_bad_request(self):
+        response = [mock.MagicMock()]
+        self._test_http_request(method=constant.GET,
+                                body=mock.sentinel.body,
+                                response=response,
+                                status_code=400)
+
+    def test_http_request_server_error(self):
+        response = [mock.MagicMock()]
+        self._test_http_request(method=constant.GET,
+                                body=mock.sentinel.body,
+                                response=response,
+                                status_code=500)
+
+    @mock.patch("hnv_client.common.utils._HNVClient._http_request")
+    def test_get_resource(self, mock_http_request):
+        response = mock.Mock()
+        response.json = mock.Mock()
+        response.json.side_effect = [mock.sentinel.response, ValueError]
+        mock_http_request.return_value = response
+
+        client = self._get_client()
+
+        self.assertIs(client.get_resource(mock.sentinel.path),
+                      mock.sentinel.response)
+        mock_http_request.assert_called_once_with(mock.sentinel.path)
+        self.assertRaises(exception.ServiceException,
+                          client.get_resource, mock.sentinel.path)
+
+    @mock.patch("hnv_client.common.utils._HNVClient._http_request")
+    def test_update_resource(self, mock_http_request):
+        response = mock.Mock()
+        response.json = mock.Mock()
+        response.json.side_effect = [mock.sentinel.response, ValueError]
+        mock_http_request.return_value = response
+
+        client = self._get_client()
+        response = client.update_resource(mock.sentinel.path,
+                                          mock.sentinel.data)
+
+        self.assertIs(response, mock.sentinel.response)
+        mock_http_request.assert_called_once_with(mock.sentinel.path,
+                                                  method="PUT",
+                                                  body=mock.sentinel.data)
+        self.assertRaises(exception.ServiceException,
+                          client.update_resource,
+                          mock.sentinel.path, mock.sentinel.data)
+
+    @mock.patch("hnv_client.common.utils._HNVClient._http_request")
+    def test_remove_resource(self, mock_http_request):
+        mock_http_request.return_value = mock.sentinel.response
+
+        client = self._get_client()
+        response = client.remove_resource(mock.sentinel.path)
+
+        self.assertIs(response, mock.sentinel.response)

setup.cfg

@@ -21,7 +21,7 @@ classifier =
 
 [files]
 packages =
-  hvn_client
+  hnv_client
 
 [global]
 setup-hooks =