diff --git a/.gitignore b/.gitignore index 88e7ec67..de469eda 100755 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ ChangeLog build/ cover/ dist +*.sqlite diff --git a/etc/refstack.conf.sample b/etc/refstack.conf.sample index 9a8b146e..7f2309f3 100644 --- a/etc/refstack.conf.sample +++ b/etc/refstack.conf.sample @@ -135,6 +135,15 @@ # The format for start_date and end_date parameters (string value) #input_date_format = %Y-%m-%d %H:%M:%S +# The GitHub API URL of the repository and location of the DefCore +# capability files. This URL is used to get a listing of all capability +# files. +#github_api_capabilities_url = https://api.github.com/repos/openstack/defcore/contents + +# The base URL that is used for retrieving specific capability files. +# Capability file names will be appended to this URL to get the contents +# of that file. +#github_raw_base_url = https://raw.githubusercontent.com/openstack/defcore/master/ [database] diff --git a/refstack/api/app.py b/refstack/api/app.py index 733c7b62..6dcaa720 100644 --- a/refstack/api/app.py +++ b/refstack/api/app.py @@ -68,6 +68,20 @@ API_OPTS = [ default='http://refstack.net/output.html?test_id=%s', help='Template for test result url.' ), + cfg.StrOpt('github_api_capabilities_url', + default='https://api.github.com' + '/repos/openstack/defcore/contents', + help='The GitHub API URL of the repository and location of ' + 'the DefCore capability files. This URL is used to get ' + 'a listing of all capability files.' + ), + cfg.StrOpt('github_raw_base_url', + default='https://raw.githubusercontent.com' + '/openstack/defcore/master/', + help='This is the base URL that is used for retrieving ' + 'specific capability files. Capability file names will ' + 'be appended to this URL to get the contents of that file.' + ) ] CONF = cfg.CONF diff --git a/refstack/api/controllers/v1.py b/refstack/api/controllers/v1.py index ae8cfc58..4620a6c9 100644 --- a/refstack/api/controllers/v1.py +++ b/refstack/api/controllers/v1.py @@ -16,10 +16,14 @@ """Version 1 of the API.""" import json + from oslo_config import cfg from oslo_log import log import pecan from pecan import rest +import re +import requests +import requests_cache from refstack import db from refstack.api import constants as const @@ -43,6 +47,8 @@ CTRLS_OPTS = [ CONF = cfg.CONF CONF.register_opts(CTRLS_OPTS, group='api') +# Cached requests will expire after 10 minutes. +requests_cache.install_cache(cache_name='github_cache', expire_after=600) class BaseRestControllerWithValidation(rest.RestController): @@ -182,8 +188,62 @@ class ResultsController(BaseRestControllerWithValidation): return page +class CapabilitiesController(rest.RestController): + + """/v1/capabilities handler. This acts as a proxy for retrieving + capability files from the openstack/defcore Github repository.""" + + @pecan.expose('json') + def get(self): + """Get a list of all available capabilities.""" + try: + response = requests.get(CONF.api.github_api_capabilities_url) + LOG.debug("Response Status: %s / Used Requests Cache: %s" % + (response.status_code, + getattr(response, 'from_cache', False))) + if response.status_code == 200: + json = response.json() + regex = re.compile('^[0-9]{4}\.[0-9]{2}\.json$') + capability_files = [] + for rfile in json: + if rfile["type"] == "file" and regex.search(rfile["name"]): + capability_files.append(rfile["name"]) + return capability_files + else: + LOG.warning('Github returned non-success HTTP ' + 'code: %s' % response.status_code) + pecan.abort(response.status_code) + + except requests.exceptions.RequestException as e: + LOG.warning('An error occurred trying to get GitHub ' + 'repository contents: %s' % e) + pecan.abort(500) + + @pecan.expose('json') + def get_one(self, file_name): + """Handler for getting contents of specific capability file.""" + github_url = ''.join((CONF.api.github_raw_base_url.rstrip('/'), + '/', file_name, ".json")) + try: + response = requests.get(github_url) + LOG.debug("Response Status: %s / Used Requests Cache: %s" % + (response.status_code, + getattr(response, 'from_cache', False))) + if response.status_code == 200: + return response.json() + else: + LOG.warning('Github returned non-success HTTP ' + 'code: %s' % response.status_code) + pecan.abort(response.status_code) + except requests.exceptions.RequestException as e: + LOG.warning('An error occurred trying to get GitHub ' + 'capability file contents: %s' % e) + pecan.abort(500) + + class V1Controller(object): """Version 1 API controller root.""" results = ResultsController() + capabilities = CapabilitiesController() diff --git a/refstack/tests/api/test_api.py b/refstack/tests/api/test_api.py index 5eb1eed4..33d9ecb1 100644 --- a/refstack/tests/api/test_api.py +++ b/refstack/tests/api/test_api.py @@ -18,6 +18,7 @@ import json import uuid +import httmock from oslo_config import fixture as config_fixture import six import webtest.app @@ -220,3 +221,37 @@ class TestResultsController(api.FunctionalTest): self.assertEqual(len(filtering_results), 3) for r in slice_results: self.assertEqual(r, filtering_results) + + +class TestCapabilitiesController(api.FunctionalTest): + """Test case for CapabilitiesController.""" + + URL = '/v1/capabilities/' + + def test_get_capability_list(self): + @httmock.all_requests + def github_api_mock(url, request): + headers = {'content-type': 'application/json'} + content = [{'name': '2015.03.json', 'type': 'file'}, + {'name': '2015.next.json', 'type': 'file'}, + {'name': '2015.03', 'type': 'dir'}] + content = json.dumps(content) + return httmock.response(200, content, headers, None, 5, request) + + with httmock.HTTMock(github_api_mock): + actual_response = self.get_json(self.URL) + + expected_response = ['2015.03.json'] + self.assertEqual(expected_response, actual_response) + + def test_get_capability_file(self): + @httmock.all_requests + def github_mock(url, request): + content = {'foo': 'bar'} + return httmock.response(200, content, None, None, 5, request) + url = self.URL + "2015.03" + with httmock.HTTMock(github_mock): + actual_response = self.get_json(url) + + expected_response = {'foo': 'bar'} + self.assertEqual(expected_response, actual_response) diff --git a/refstack/tests/unit/test_api.py b/refstack/tests/unit/test_api.py index 614412dd..93a19a04 100644 --- a/refstack/tests/unit/test_api.py +++ b/refstack/tests/unit/test_api.py @@ -15,9 +15,14 @@ """Tests for API's controllers""" +import json +import sys + +import httmock import mock from oslo_config import fixture as config_fixture from oslotest import base +import requests from refstack.api import constants as const from refstack.api import utils as api_utils @@ -25,6 +30,15 @@ from refstack.api.controllers import root from refstack.api.controllers import v1 +def safe_json_dump(content): + if isinstance(content, (dict, list)): + if sys.version_info[0] == 3: + content = bytes(json.dumps(content), 'utf-8') + else: + content = json.dumps(content) + return content + + class RootControllerTestCase(base.BaseTestCase): def test_index(self): @@ -230,6 +244,81 @@ class ResultsControllerTestCase(base.BaseTestCase): db_get_test.assert_called_once_with(page_number, per_page, filters) +class CapabilitiesControllerTestCase(base.BaseTestCase): + + def setUp(self): + super(CapabilitiesControllerTestCase, self).setUp() + self.controller = v1.CapabilitiesController() + + def test_get_capabilities(self): + """Test when getting a list of all capability files.""" + @httmock.all_requests + def github_api_mock(url, request): + headers = {'content-type': 'application/json'} + content = [{'name': '2015.03.json', 'type': 'file'}, + {'name': '2015.next.json', 'type': 'file'}, + {'name': '2015.03', 'type': 'dir'}] + content = safe_json_dump(content) + return httmock.response(200, content, headers, None, 5, request) + + with httmock.HTTMock(github_api_mock): + result = self.controller.get() + self.assertEqual(['2015.03.json'], result) + + @mock.patch('pecan.abort') + def test_get_capabilities_error_code(self, mock_abort): + """Test when the HTTP status code isn't a 200 OK. The status should + be propogated.""" + @httmock.all_requests + def github_api_mock(url, request): + content = {'title': 'Not Found'} + return httmock.response(404, content, None, None, 5, request) + + with httmock.HTTMock(github_api_mock): + self.controller.get() + mock_abort.assert_called_with(404) + + @mock.patch('requests.get') + @mock.patch('pecan.abort') + def test_get_capabilities_exception(self, mock_abort, mock_request): + """Test when the GET request raises an exception.""" + mock_request.side_effect = requests.exceptions.RequestException() + self.controller.get() + mock_abort.assert_called_with(500) + + def test_get_capability_file(self): + """Test when getting a specific capability file""" + @httmock.all_requests + def github_mock(url, request): + content = {'foo': 'bar'} + return httmock.response(200, content, None, None, 5, request) + + with httmock.HTTMock(github_mock): + result = self.controller.get_one('2015.03') + self.assertEqual({'foo': 'bar'}, result) + + @mock.patch('pecan.abort') + def test_get_capability_file_error_code(self, mock_abort): + """Test when the HTTP status code isn't a 200 OK. The status should + be propogated.""" + @httmock.all_requests + def github_api_mock(url, request): + content = {'title': 'Not Found'} + return httmock.response(404, content, None, None, 5, request) + + with httmock.HTTMock(github_api_mock): + self.controller.get_one('2010.03') + mock_abort.assert_called_with(404) + + @mock.patch('requests.get') + @mock.patch('pecan.abort') + def test_get_capability_file_exception(self, mock_abort, mock_request): + """Test when the GET request raises an exception.""" + mock_request.side_effect = requests.exceptions.RequestException() + self.controller.get_one('2010.03') + mock_abort.assert_called_with(500) + + class BaseRestControllerWithValidationTestCase(base.BaseTestCase): def setUp(self): diff --git a/requirements.txt b/requirements.txt index 4a762616..96ff43c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ oslo.log pecan>=0.8.2 pyOpenSSL==0.13 pycrypto>=2.6 -requests==1.2.3 +requests>=2.2.0,!=2.4.0 +requests-cache>=0.4.9 jsonschema>=2.0.0,<3.0.0 PyMySQL>=0.6.2,!=0.6.4 \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt index 443e1a73..c700e9e1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,6 +2,7 @@ coverage>=3.6 pep8==1.5.7 pyflakes==0.8.1 flake8==2.2.4 +httmock mock oslotest>=1.2.0 # Apache-2.0 python-subunit>=0.0.18