From 68c6f2cc3ebf1b5ab96707694bee6b492fe9202c Mon Sep 17 00:00:00 2001 From: Michal Ptacek Date: Wed, 2 Nov 2016 15:14:43 +0000 Subject: [PATCH] Adding Keystone V3 API support in master branch keystone v2.0 api is no longer supported, this patch is introducing v3 api support. Change-Id: I5ed5f65f34033b6a4c550704bb186dfa8d0fc82c Closes-Bug: #1614892 --- collectd_ceilometer/keystone_light.py | 107 ++++++ collectd_ceilometer/sender.py | 21 +- collectd_ceilometer/settings.py | 1 + collectd_ceilometer/tests/base.py | 1 + .../tests/test_keystone_light.py | 333 +++++++++++++++++- devstack/libs/collectd | 1 + devstack/settings | 1 + 7 files changed, 456 insertions(+), 9 deletions(-) diff --git a/collectd_ceilometer/keystone_light.py b/collectd_ceilometer/keystone_light.py index e49d30b..a65ce32 100644 --- a/collectd_ceilometer/keystone_light.py +++ b/collectd_ceilometer/keystone_light.py @@ -13,8 +13,11 @@ # under the License. """ Lightweight (keystone) client for the OpenStack Identity API """ +import logging import requests +LOG = logging.getLogger(__name__) + class KeystoneException(Exception): def __init__(self, message, exc=None, response=None): @@ -38,6 +41,110 @@ class MissingServices(KeystoneException): "MissingServices: " + message, exc, response) +class ClientV3(object): + """Light weight client for the OpenStack Identity API V3. + + :param string username: Username for authentication. + :param string password: Password for authentication. + :param string tenant_name: Tenant name. + :param string auth_url: Keystone service endpoint for authorization. + + """ + + def __init__(self, auth_url, username, password, tenant_name): + """Initialize a new client""" + + self.auth_url = auth_url + self.username = username + self.password = password + self.tenant_name = tenant_name + self._auth_token = None + self._services = () + self._services_by_name = {} + + @property + def auth_token(self): + """Return token string usable for X-Auth-Token """ + # actualize token + self.refresh() + return self._auth_token + + @property + def services(self): + """Return list of services retrieved from identity server """ + return self._services + + def refresh(self): + """Refresh token and services list (getting it from identity server) """ + headers = {'Accept': 'application/json'} + url = self.auth_url.rstrip('/') + '/auth/tokens' + params = { + 'auth': { + 'identity': { + 'methods': ['password'], + 'password': { + 'user': { + 'name': self.username, + 'domain': {'id': 'default'}, + 'password': self.password + } + } + }, + 'scope': { + 'project': { + 'name': self.tenant_name, + 'domain': {'id': 'default'} + } + } + } + } + + resp = requests.post(url, json=params, headers=headers) + resp_data = None + # processing response + try: + resp.raise_for_status() + resp_data = resp.json()['token'] + self._services = tuple(resp_data['catalog']) + self._services_by_name = { + service['name']: service for service in self._services + } + self._auth_token = resp.headers['X-Subject-Token'] + except (TypeError, KeyError, ValueError, + requests.exceptions.HTTPError) as e: + LOG.exception("Error processing response from keystone") + raise InvalidResponse(e, resp_data) + return resp_data + + def get_service_endpoint(self, name, urlkey="internalURL", region=None): + """Return url endpoint of service + + possible values of urlkey = 'adminURL' | 'publicURL' | 'internalURL' + provide region if more endpoints are available + """ + + try: + endpoints = self._services_by_name[name]['endpoints'] + if not endpoints: + raise MissingServices("Missing name '%s' in received services" + % name, + None, self._services) + + if region: + for ep in endpoints: + if ep['region'] == region and ep['interface'] in urlkey: + return ep["url"].rstrip('/') + else: + for ep in endpoints: + if ep['interface'] in urlkey: + return ep["url"].rstrip('/') + raise MissingServices("No valid endpoints found") + except (KeyError, ValueError) as e: + LOG.exception("Error while processing endpoints") + raise MissingServices("Missing data in received services", + e, self._services) + + class ClientV2(object): """Light weight client for the OpenStack Identity API V2. diff --git a/collectd_ceilometer/sender.py b/collectd_ceilometer/sender.py index a9f2dec..8c00f42 100644 --- a/collectd_ceilometer/sender.py +++ b/collectd_ceilometer/sender.py @@ -23,6 +23,7 @@ import requests import six from collectd_ceilometer.keystone_light import ClientV2 +from collectd_ceilometer.keystone_light import ClientV3 from collectd_ceilometer.keystone_light import KeystoneException from collectd_ceilometer.settings import Config @@ -77,12 +78,20 @@ class Sender(object): # create a keystone client if it doesn't exist if self._keystone is None: cfg = Config.instance() - self._keystone = ClientV2( - auth_url=cfg.OS_AUTH_URL, - username=cfg.OS_USERNAME, - password=cfg.OS_PASSWORD, - tenant_name=cfg.OS_TENANT_NAME - ) + if cfg.OS_IDENTITY_API_VERSION == "2.0": + self._keystone = ClientV2( + auth_url=cfg.OS_AUTH_URL, + username=cfg.OS_USERNAME, + password=cfg.OS_PASSWORD, + tenant_name=cfg.OS_TENANT_NAME + ) + else: + self._keystone = ClientV3( + auth_url=cfg.OS_AUTH_URL, + username=cfg.OS_USERNAME, + password=cfg.OS_PASSWORD, + tenant_name=cfg.OS_TENANT_NAME + ) # store the authentication token self._auth_token = self._keystone.auth_token diff --git a/collectd_ceilometer/settings.py b/collectd_ceilometer/settings.py index 51bc0ae..c41b7b1 100644 --- a/collectd_ceilometer/settings.py +++ b/collectd_ceilometer/settings.py @@ -51,6 +51,7 @@ class Config(object): _configuration = [ CfgParam('BATCH_SIZE', 1, int), CfgParam('OS_AUTH_URL', None, six.text_type), + CfgParam('OS_IDENTITY_API_VERSION', '3', six.text_type), CfgParam('CEILOMETER_URL_TYPE', 'internalURL', six.text_type), CfgParam('CEILOMETER_TIMEOUT', 1000, int), CfgParam('OS_USERNAME', None, six.text_type), diff --git a/collectd_ceilometer/tests/base.py b/collectd_ceilometer/tests/base.py index 69e4ae1..5ac40b2 100644 --- a/collectd_ceilometer/tests/base.py +++ b/collectd_ceilometer/tests/base.py @@ -50,6 +50,7 @@ class TestConfig(object): default_values = OrderedDict([ ('BATCH_SIZE', 1,), + ('OS_IDENTITY_API_VERSION', '2.0'), ('OS_AUTH_URL', 'https://test-auth.url.tld/test',), ('CEILOMETER_URL_TYPE', 'internalURL',), ('CEILOMETER_TIMEOUT', 1000,), diff --git a/collectd_ceilometer/tests/test_keystone_light.py b/collectd_ceilometer/tests/test_keystone_light.py index 99b9983..31de53e 100644 --- a/collectd_ceilometer/tests/test_keystone_light.py +++ b/collectd_ceilometer/tests/test_keystone_light.py @@ -20,16 +20,343 @@ from __future__ import unicode_literals from collectd_ceilometer import keystone_light from collectd_ceilometer.keystone_light import ClientV2 +from collectd_ceilometer.keystone_light import ClientV3 from collectd_ceilometer.keystone_light import MissingServices import mock import unittest -class KeystoneLightTest(unittest.TestCase): - """Test the keystone light client""" +class KeystoneLightTestV3(unittest.TestCase): + """Test the keystone light client with 3.0 keystone api""" def setUp(self): - super(KeystoneLightTest, self).setUp() + super(KeystoneLightTestV3, self).setUp() + + self.test_authtoken = "c5bbb1c9a27e470fb482de2a718e08c2" + self.test_public_endpoint = "http://public_endpoint" + self.test_internal_endpoint = "http://iternal_endpoint" + self.test_region = "RegionOne" + + response = {"token": { + "is_domain": 'false', + "methods": [ + "password" + ], + "roles": [ + { + "id": "eacf519eb1264cba9ad645355ce1f6ec", + "name": "ResellerAdmin" + }, + { + "id": "63e481b5d5f545ecb8947072ff34f10d", + "name": "admin" + } + ], + "is_admin_project": 'false', + "project": { + "domain": { + "id": "default", + "name": "Default" + }, + "id": "97467f21efb2493c92481429a04df7bd", + "name": "service" + }, + "catalog": [ + { + "endpoints": [ + { + "url": self.test_public_endpoint + '/', + "interface": "public", + "region": "RegionOne", + "region_id": self.test_region, + "id": "5e1d9a45d7d442ca8971a5112b2e89b5" + }, + { + "url": "http://127.0.0.1:8777", + "interface": "admin", + "region": "RegionOne", + "region_id": self.test_region, + "id": "5e8b536fde6049d381ee540c018905d1" + }, + { + "url": self.test_internal_endpoint + '/', + "interface": "internal", + "region": "RegionOne", + "region_id": self.test_region, + "id": "db90c733ddd9466696bc5aaec43b18d0" + } + ], + "type": "metering", + "id": "f6c15a041d574bc190c70815a14ab851", + "name": "ceilometer" + } + ] + } + } + + self.mock_response = mock.Mock() + self.mock_response.json.return_value = response + self.mock_response.headers = { + 'X-Subject-Token': "c5bbb1c9a27e470fb482de2a718e08c2" + } + + @mock.patch('collectd_ceilometer.keystone_light.requests.post') + def test_refresh(self, mock_post): + """Test refresh""" + + mock_post.return_value = self.mock_response + + client = ClientV3("test_auth_url", "test_username", + "test_password", "test_tenant") + + self.assertEqual(client.auth_token, self.test_authtoken) + + expected_args = { + 'headers': {'Accept': 'application/json'}, + 'json': { + 'auth': { + "identity": { + "methods": ["password"], + "password": { + "user": { + "name": u'test_username', + "domain": {"id": "default"}, + "password": u'test_password' + } + } + }, + "scope": { + "project": { + "name": u'test_tenant', + "domain": {"id": "default"} + } + } + } + } + } + + mock_post.assert_called_once_with( + 'test_auth_url/auth/tokens', + json=expected_args['json'], + headers=expected_args['headers'], + ) + + @mock.patch('collectd_ceilometer.keystone_light.requests.post') + def test_getservice_endpoint(self, mock_post): + """Test getservice endpoint""" + + mock_post.return_value = self.mock_response + + client = ClientV3("test_auth_url", "test_username", + "test_password", "test_tenant") + client.refresh() + + endpoint = client.get_service_endpoint('ceilometer') + self.assertEqual(endpoint, self.test_internal_endpoint) + + endpoint = client.get_service_endpoint('ceilometer', 'publicURL') + self.assertEqual(endpoint, self.test_public_endpoint) + + endpoint = client.get_service_endpoint('ceilometer', 'publicURL', + self.test_region) + self.assertEqual(endpoint, self.test_public_endpoint) + + with self.assertRaises(MissingServices): + client.get_service_endpoint('badname') + + @mock.patch('collectd_ceilometer.keystone_light.requests.post') + def test_getservice_endpoint_error(self, mock_post): + """Test getservice endpoint error""" + + response = {"token": { + "is_domain": 'false', + "methods": [ + "password" + ], + "roles": [ + { + "id": "eacf519eb1264cba9ad645355ce1f6ec", + "name": "ResellerAdmin" + }, + { + "id": "63e481b5d5f545ecb8947072ff34f10d", + "name": "admin" + } + ], + "is_admin_project": 'false', + "project": { + "domain": { + "id": "default", + "name": "Default" + }, + "id": "97467f21efb2493c92481429a04df7bd", + "name": "service" + }, + "catalog": [ + { + "endpoints": [], + "type": "metering", + "id": "f6c15a041d574bc190c70815a14ab851", + "name": "badname" + } + ] + } + } + + self.mock_response = mock.Mock() + self.mock_response.json.return_value = response + self.mock_response.headers = { + 'X-Subject-Token': "c5bbb1c9a27e470fb482de2a718e08c2" + } + mock_post.return_value = self.mock_response + + client = ClientV3("test_auth_url", "test_username", + "test_password", "test_tenant") + client.refresh() + + with self.assertRaises(MissingServices): + client.get_service_endpoint('ceilometer') + + @mock.patch('collectd_ceilometer.keystone_light.requests.post') + def test_invalidresponse_missing_token(self, mock_post): + """Test invalid response: missing access""" + + response = {'badresponse': None} + + mock_response = mock.Mock() + mock_response.json.return_value = response + mock_response.headers = { + 'X-Subject-Token': "c5bbb1c9a27e470fb482de2a718e08c2" + } + mock_post.return_value = mock_response + + client = keystone_light.ClientV3("test_auth_url", "test_username", + "test_password", "test_tenant") + + with self.assertRaises(keystone_light.InvalidResponse): + client.refresh() + + @mock.patch('collectd_ceilometer.keystone_light.requests.post') + def test_invalidresponse_missing_catalog(self, mock_post): + """Test invalid response: missing catalog""" + + response = {"token": { + "is_domain": 'false', + "methods": [ + "password" + ], + "roles": [ + { + "id": "eacf519eb1264cba9ad645355ce1f6ec", + "name": "ResellerAdmin" + }, + { + "id": "63e481b5d5f545ecb8947072ff34f10d", + "name": "admin" + } + ], + "is_admin_project": 'false', + "project": { + "domain": { + "id": "default", + "name": "Default" + }, + "id": "97467f21efb2493c92481429a04df7bd", + "name": "service" + }, + } + } + + mock_response = mock.Mock() + mock_response.json.return_value = response + mock_response.headers = { + 'X-Subject-Token': "c5bbb1c9a27e470fb482de2a718e08c2" + } + mock_post.return_value = mock_response + + client = keystone_light.ClientV3("test_auth_url", "test_username", + "test_password", "test_tenant") + + with self.assertRaises(keystone_light.InvalidResponse): + client.refresh() + + @mock.patch('collectd_ceilometer.keystone_light.requests.post') + def test_invalidresponse_missing_token_http_header(self, mock_post): + """Test invalid response: missing token in header""" + + response = {"token": { + "is_domain": 'false', + "methods": [ + "password" + ], + "roles": [ + { + "id": "eacf519eb1264cba9ad645355ce1f6ec", + "name": "ResellerAdmin" + }, + { + "id": "63e481b5d5f545ecb8947072ff34f10d", + "name": "admin" + } + ], + "is_admin_project": 'false', + "project": { + "domain": { + "id": "default", + "name": "Default" + }, + "id": "97467f21efb2493c92481429a04df7bd", + "name": "service" + }, + "catalog": [ + { + "endpoints": [ + { + "url": self.test_public_endpoint + '/', + "interface": "public", + "region": "RegionOne", + "region_id": self.test_region, + "id": "5e1d9a45d7d442ca8971a5112b2e89b5" + }, + { + "url": "http://127.0.0.1:8777", + "interface": "admin", + "region": "RegionOne", + "region_id": self.test_region, + "id": "5e8b536fde6049d381ee540c018905d1" + }, + { + "url": self.test_internal_endpoint + '/', + "interface": "internal", + "region": "RegionOne", + "region_id": self.test_region, + "id": "db90c733ddd9466696bc5aaec43b18d0" + } + ], + "type": "metering", + "id": "f6c15a041d574bc190c70815a14ab851", + "name": "ceilometer" + } + ] + } + } + + mock_response = mock.Mock() + mock_response.json.return_value = response + mock_post.return_value = mock_response + + client = keystone_light.ClientV3("test_auth_url", "test_username", + "test_password", "test_tenant") + + with self.assertRaises(keystone_light.InvalidResponse): + client.refresh() + + +class KeystoneLightTestV2(unittest.TestCase): + """Test the keystone light client with 2.0 keystone api""" + + def setUp(self): + super(KeystoneLightTestV2, self).setUp() self.test_authtoken = "c5bbb1c9a27e470fb482de2a718e08c2" self.test_public_endpoint = "http://public_endpoint" diff --git a/devstack/libs/collectd b/devstack/libs/collectd index a8cf780..bf83ed7 100644 --- a/devstack/libs/collectd +++ b/devstack/libs/collectd @@ -55,6 +55,7 @@ cat << EOF | sudo tee $COLLECTD_CONF_DIR/collectd-ceilometer-plugin.conf # Service endpoint addresses OS_AUTH_URL "$OS_AUTH_URL" + OS_IDENTITY_API_VERSION "$OS_IDENTITY_API_VERSION" # Ceilometer address #CEILOMETER_ENDPOINT diff --git a/devstack/settings b/devstack/settings index 068677c..91edeb2 100644 --- a/devstack/settings +++ b/devstack/settings @@ -16,6 +16,7 @@ CEILOMETER_TIMEOUT=${CEILOMETER_TIMEOUT:-1000} # Auth info OS_AUTH_URL="$KEYSTONE_AUTH_URI/v$IDENTITY_API_VERSION" +OS_IDENTITY_API_VERSION=${IDENTITY_API_VERSION:-3} # Fall back to default conf dir if option is unset if [ -z $COLLECTD_CONF_DIR ]; then