From 8bf79c3b6ad2bf065cb543b47efd703eba30de37 Mon Sep 17 00:00:00 2001 From: Alexandru Coman Date: Mon, 30 Jan 2017 22:01:59 +0200 Subject: [PATCH] Add basic client for HVN REST API --- .pylintrc | 2 +- hnv_client/common/constant.py | 19 ++ hnv_client/common/exception.py | 32 +++- hnv_client/common/utils.py | 202 ++++++++++++++++++++ hnv_client/config/hnv.py | 12 ++ hnv_client/tests/common/test_utils.py | 253 ++++++++++++++++++++++++++ setup.cfg | 2 +- 7 files changed, 517 insertions(+), 5 deletions(-) create mode 100644 hnv_client/common/utils.py create mode 100644 hnv_client/tests/common/test_utils.py diff --git a/.pylintrc b/.pylintrc index a73c8b3..83db279 100644 --- a/.pylintrc +++ b/.pylintrc @@ -99,7 +99,7 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO +notes=FIXME,XXX [FORMAT] diff --git a/hnv_client/common/constant.py b/hnv_client/common/constant.py index c4dd331..ddf1d41 100644 --- a/hnv_client/common/constant.py +++ b/hnv_client/common/constant.py @@ -13,3 +13,22 @@ # under the License. """Shared constants across the bcbio-nextgen-vm project.""" + +GET = "GET" +POST = "POST" +PUT = "PUT" +PATCH = "PATCH" +DELETE = "DELETE" + +DELETING = "Deleting" +FAILED = "Failed" +SUCCEEDED = "Succeeded" +UPDATING = "Updating" + +ABSOLUTE = "absolute" +WEIGHT = "weight" + +VIRTUAL_APPLIANCE = "VirtualAppliance" +VNET_LOCAL = "VnetLocal" +VIRTUAL_NETWORK_GATEWAY = "VirtualNetworkGateway" +INTERNET = "Internet" diff --git a/hnv_client/common/exception.py b/hnv_client/common/exception.py index fc89e27..c1855c4 100644 --- a/hnv_client/common/exception.py +++ b/hnv_client/common/exception.py @@ -58,15 +58,41 @@ class DataProcessingError(HNVException): template = "The provided information is incomplete or invalid." -class NotFound(HNVException): +class ServiceException(HNVException): + + """Base exception for all the API interaction related errors.""" + + template = "Something went wrong." + + +class TimeOut(ServiceException): + + """The request timed out.""" + + template = "The request timed out." + + +class NotFound(ServiceException): """The required object is not available in container.""" template = "The %(object)r was not found in %(container)s." -class NotSupported(HNVException): +class CertificateVerifyFailed(ServiceException): + + """The received certificate is not valid. + + In order to avoid the current exception the validation of the SSL + certificate should be disabled for the metadata provider. In order + to do that the `https_allow_insecure` config option should be set. + """ + + template = "The received certificate is not valid." + + +class NotSupported(ServiceException): """The functionality required is not available in the current context.""" - template = "%(feature)s is not available in %(context)s." + template = "%(feature)s is not available for %(context)s." diff --git a/hnv_client/common/utils.py b/hnv_client/common/utils.py new file mode 100644 index 0000000..5f38617 --- /dev/null +++ b/hnv_client/common/utils.py @@ -0,0 +1,202 @@ +# Copyright 2017 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utilities used across the project.""" + +import json +import sys +import time + +from oslo_log import log as logging +import requests +import requests_ntlm +import six + +from hnv_client.common import constant +from hnv_client.common import exception +from hnv_client import config as hnv_config + +LOG = logging.getLogger(__name__) +CONFIG = hnv_config.CONFIG + + +class _HNVClient(object): + + """Minimalistic client for the Network Controller REST API. + + :param url: The base URL where the agent looks for + Network Controller API. + :param username: The username required for connecting to the + Network Controller API. + :param password: The password required for connecting to the + Network Controller API. + :param allow_insecure: Whether to disable the validation of + HTTPS certificates. + :param ca_bundle: The path to a CA_BUNDLE file or directory + with certificates of trusted CAs. + """ + + def __init__(self, url, username=None, password=None, allow_insecure=False, + ca_bundle=None): + self._base_url = url + self._credentials = (username, password) + self._https_allow_insecure = allow_insecure + self._https_ca_bundle = ca_bundle + self._http_session = None + + @property + def _session(self): + """The current session used by the client. + + The Session object allows you to persist certain parameters across + requests. It also persists cookies across all requests made from + the Session instance, and will use urllib3's connection pooling. + So if you're making several requests to the same host, the underlying + TCP connection will be reused, which can result in a significant + performance increase. + """ + if self._http_session is None: + self._http_session = requests.Session() + self._http_session.headers.update(self._get_headers()) + self._http_session.verify = self._verify_https_request() + + if all(self._credentials): + username, password = self._credentials + self._http_session.auth = requests_ntlm.HttpNtlmAuth( + username=username, password=password) + + return self._http_session + + @staticmethod + def _get_headers(): + """Prepare the HTTP headers for the current request.""" + + # TODO(alexcoman): Add the x-ms-client-ip-address header in order + # to improve the Network Controller requests logging. + return { + "Accept": "application/json", + "Connection": "keep-alive", + "Content-Type": "application/json; charset=UTF-8", + } + + def _verify_https_request(self): + """Whether to disable the validation of HTTPS certificates. + + .. notes:: + When `https_allow_insecure` option is `True` the SSL certificate + validation for the connection with the Network Controller API will + be disabled (please don't use it if you don't know the + implications of this behaviour). + """ + if self._https_ca_bundle: + return self._https_ca_bundle + else: + return not self._https_allow_insecure + + def _http_request(self, resource, method=constant.GET, body=None): + url = requests.compat.urljoin(self._base_url, resource) + headers = self._get_headers() + if method in (constant.PUT, constant.PATCH): + etag = (body or {}).get("etag", None) + if etag is not None: + headers["If-Match"] = etag + + attemts = 0 + while True: + try: + response = self._session.request( + method=method, url=url, headers=headers, + data=json.dumps(body) if body else None, + timeout=CONFIG.HNV.http_request_timeout + ) + break + except requests.ConnectionError as exc: + attemts += 1 + self._http_session = None + LOG.debug("Request failed: %s", exc) + if attemts > CONFIG.HNV.retry_count: + if isinstance(exc, requests.exceptions.SSLError): + raise exception.CertificateVerifyFailed( + "HTTPS certificate validation failed.") + raise + time.sleep(CONFIG.HNV.retry_interval) + + try: + response.raise_for_status() + except requests.HTTPError as exc: + status_code = exc.response.status_code + content = exc.response.text + LOG.debug("HTTP Error %(status_code)r: %(details)r", + {"status_code": status_code, "details": content}) + + if status_code == 400: + raise exception.ServiceException( + ("HNV Client failed to communicate with the API. " + "Please open an issue with the following information: " + "%(resource)r: %(details)r"), + resource=resource, details=content + ) + if status_code == 404: + raise exception.NotFound( + "Resource %(resource)r was not found.", resource=resource) + raise + + return response + + def get_resource(self, path): + """Getting the required information from the API.""" + response = self._http_request(path) + try: + return response.json() + except ValueError: + raise exception.ServiceException("Invalid service response.") + + def update_resource(self, path, data): + """Update the required resource.""" + response = self._http_request(path, method="PUT", body=data) + try: + return response.json() + except ValueError: + raise exception.ServiceException("Invalid service response.") + + def remove_resource(self, path): + """Delete the received resource.""" + return self._http_request(path, method="DELETE") + + +# pylint: disable=dangerous-default-value +def run_once(function, state={}, errors={}): + """A memoization decorator, whose purpose is to cache calls.""" + @six.wraps(function) + def _wrapper(*args, **kwargs): + if function in errors: + # Deliberate use of LBYL. + six.reraise(*errors[function]) + + try: + return state[function] + except KeyError: + try: + state[function] = result = function(*args, **kwargs) + return result + except Exception: + errors[function] = sys.exc_info() + raise + return _wrapper + + +@run_once +def get_client(url, username, password, allow_insecure, ca_bundle): + """Create a new client for the HNV REST API.""" + return _HNVClient(url, username, password, allow_insecure, ca_bundle) diff --git a/hnv_client/config/hnv.py b/hnv_client/config/hnv.py index 0ac639f..eae8927 100644 --- a/hnv_client/config/hnv.py +++ b/hnv_client/config/hnv.py @@ -47,10 +47,22 @@ class HVNOptions(config_base.Options): "https_ca_bundle", default=None, help=("The path to a CA_BUNDLE file or directory with " "certificates of trusted CAs.")), + cfg.IntOpt( + "retry_count", default=5, + help="Max. number of attempts for fetching metadata in " + "case of transient errors"), cfg.FloatOpt( "retry_interval", default=1, help=("Interval between attempts in case of transient errors, " "expressed in seconds")), + cfg.IntOpt( + "http_request_timeout", default=None, + help=("Number of seconds until network requests stop waiting " + "for a response")), + cfg.StrOpt( + "logical_network", default=None, + help=("Logical network to use as a medium for tenant network " + "traffic.")), ] def register(self): diff --git a/hnv_client/tests/common/test_utils.py b/hnv_client/tests/common/test_utils.py new file mode 100644 index 0000000..43b9a86 --- /dev/null +++ b/hnv_client/tests/common/test_utils.py @@ -0,0 +1,253 @@ +# Copyright 2017 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# pylint: disable=protected-access, missing-docstring + +import unittest +try: + import unittest.mock as mock +except ImportError: + import mock + +import requests + +from hnv_client.common import constant +from hnv_client.common import exception +from hnv_client.common import utils as hnv_utils +from hnv_client import config as hnv_config +from hnv_client.tests import utils as test_utils + +CONFIG = hnv_config.CONFIG + + +class TestHNVClient(unittest.TestCase): + + @staticmethod + def _get_client(url=mock.sentinel.url, username=mock.sentinel.username, + password=mock.sentinel.password, + allow_insecure=mock.sentinel.insecure, + ca_bundle=mock.sentinel.ca_bundle): + return hnv_utils._HNVClient(url, username, password, allow_insecure, + ca_bundle) + + @mock.patch("hnv_client.common.utils._HNVClient._get_headers") + @mock.patch("hnv_client.common.utils._HNVClient._verify_https_request") + @mock.patch("requests_ntlm.HttpNtlmAuth") + @mock.patch("requests.Session") + def test_session(self, mock_get_session, mock_auth, mock_verify, + mock_headers): + mock_session = mock.Mock() + mock_session.headers = {} + mock_get_session.return_value = mock_session + mock_verify.return_value = mock.sentinel.verify + mock_auth.return_value = mock.sentinel.auth + mock_headers.return_value = {"X-HNV-Test": 1} + + client = self._get_client() + session = client._session + + self.assertIs(session, mock_session) + self.assertIs(mock_session.verify, mock.sentinel.verify) + self.assertIs(mock_session.auth, mock.sentinel.auth) + self.assertEqual(mock_session.headers.get("X-HNV-Test"), 1) + mock_auth.assert_called_once_with(username=mock.sentinel.username, + password=mock.sentinel.password) + + def test_verify_https_request(self): + ca_bundle_client = self._get_client(allow_insecure=None) + insecure_client = self._get_client(ca_bundle=None) + + self.assertIs(ca_bundle_client._verify_https_request(), + mock.sentinel.ca_bundle) + self.assertFalse(insecure_client._verify_https_request()) + + @mock.patch("time.sleep") + @mock.patch("json.dumps") + @mock.patch("requests.compat.urljoin") + @mock.patch("hnv_client.common.utils._HNVClient._session") + @mock.patch("hnv_client.common.utils._HNVClient._get_headers") + def _test_http_request(self, mock_headers, mock_session, mock_join, + mock_dump, mock_sleep, + method, body, response, status_code): + output = [] + headers = mock_headers.return_value = {} + mock_join.return_value = mock.sentinel.url + mock_dump.return_value = mock.sentinel.content + + session_request = mock_session.request = mock.MagicMock() + session_request.side_effect = response + + expected_response = response[-1] + + status_check = expected_response.raise_for_status = mock.MagicMock() + if status_code != 200: + exc_response = mock.MagicMock() + exc_response.status_code = status_code + exc_response.text = "Expected Error" + status_check.side_effect = requests.HTTPError( + response=exc_response) + output.append("HTTP Error %(status_code)r: 'Expected Error'" % + {"status_code": status_code}) + + client = self._get_client() + with test_utils.LogSnatcher("hnv_client.common.utils") as logging: + if isinstance(expected_response, requests.exceptions.SSLError): + self.assertRaises(exception.CertificateVerifyFailed, + client._http_request, + mock.sentinel.resource, method, body) + return + elif isinstance(expected_response, requests.ConnectionError): + self.assertRaises(requests.ConnectionError, + client._http_request, + mock.sentinel.resource, method, body) + return + elif status_code == 400: + self.assertRaises(exception.ServiceException, + client._http_request, + mock.sentinel.resource, method, body) + elif status_code == 404: + self.assertRaises(exception.NotFound, + client._http_request, + mock.sentinel.resource, method, body) + elif status_code != 200: + self.assertRaises(requests.HTTPError, + client._http_request, + mock.sentinel.resource, method, body) + else: + client_response = client._http_request(mock.sentinel.resource, + method, body) + + mock_join.assert_called_once_with(mock.sentinel.url, + mock.sentinel.resource) + mock_headers.assert_called_once_with() + if not method == constant.GET: + etag = (body or {}).get("etag", None) + self.assertEqual(headers["If-Match"], etag) + + if len(response) == 1: + session_request.assert_called_once_with( + method=method, url=mock.sentinel.url, headers=headers, + data=mock.sentinel.content if body else None, + timeout=CONFIG.HNV.http_request_timeout + ) + elif len(response) > 1: + # Note(alexcoman): The first response is an exception + output.append("Request failed: ") + + self.assertEqual(logging.output, output) + if status_code == 200: + self.assertIs(client_response, expected_response) + + def test_http_request_get(self): + response = [mock.MagicMock()] + self._test_http_request(method=constant.GET, + body=mock.sentinel.body, + response=response, + status_code=200) + + def test_http_request_put(self): + response = [mock.MagicMock()] + self._test_http_request(method=constant.PUT, + body={"etag": mock.sentinel.etag}, + response=response, + status_code=200) + + def test_http_request_with_connection_error(self): + response = [requests.ConnectionError(), mock.MagicMock()] + with test_utils.ConfigPatcher('retry_count', 1, "HNV"): + self._test_http_request(method=constant.GET, + body=mock.sentinel.body, + response=response, + status_code=200) + + def test_http_request_connection_error(self): + response = [requests.ConnectionError(), requests.ConnectionError()] + with test_utils.ConfigPatcher('retry_count', 1, "HNV"): + self._test_http_request(method=constant.GET, + body=mock.sentinel.body, + response=response, + status_code=200) + + def test_http_request_ssl_error(self): + response = [requests.exceptions.SSLError(), + requests.exceptions.SSLError()] + with test_utils.ConfigPatcher('retry_count', 1, "HNV"): + self._test_http_request(method=constant.GET, + body=mock.sentinel.body, + response=response, + status_code=200) + + def test_http_request_not_found(self): + response = [mock.MagicMock()] + self._test_http_request(method=constant.GET, + body=mock.sentinel.body, + response=response, + status_code=404) + + def test_http_request_bad_request(self): + response = [mock.MagicMock()] + self._test_http_request(method=constant.GET, + body=mock.sentinel.body, + response=response, + status_code=400) + + def test_http_request_server_error(self): + response = [mock.MagicMock()] + self._test_http_request(method=constant.GET, + body=mock.sentinel.body, + response=response, + status_code=500) + + @mock.patch("hnv_client.common.utils._HNVClient._http_request") + def test_get_resource(self, mock_http_request): + response = mock.Mock() + response.json = mock.Mock() + response.json.side_effect = [mock.sentinel.response, ValueError] + mock_http_request.return_value = response + + client = self._get_client() + + self.assertIs(client.get_resource(mock.sentinel.path), + mock.sentinel.response) + mock_http_request.assert_called_once_with(mock.sentinel.path) + self.assertRaises(exception.ServiceException, + client.get_resource, mock.sentinel.path) + + @mock.patch("hnv_client.common.utils._HNVClient._http_request") + def test_update_resource(self, mock_http_request): + response = mock.Mock() + response.json = mock.Mock() + response.json.side_effect = [mock.sentinel.response, ValueError] + mock_http_request.return_value = response + + client = self._get_client() + response = client.update_resource(mock.sentinel.path, + mock.sentinel.data) + + self.assertIs(response, mock.sentinel.response) + mock_http_request.assert_called_once_with(mock.sentinel.path, + method="PUT", + body=mock.sentinel.data) + self.assertRaises(exception.ServiceException, + client.update_resource, + mock.sentinel.path, mock.sentinel.data) + + @mock.patch("hnv_client.common.utils._HNVClient._http_request") + def test_remove_resource(self, mock_http_request): + mock_http_request.return_value = mock.sentinel.response + + client = self._get_client() + response = client.remove_resource(mock.sentinel.path) + + self.assertIs(response, mock.sentinel.response) diff --git a/setup.cfg b/setup.cfg index 29c7ce7..78dba6e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,7 +21,7 @@ classifier = [files] packages = - hvn_client + hnv_client [global] setup-hooks =