Use keystoneauth1 instead of manual setup

This change moves our code the the new keystoneauth1 library.

This allows to wipe out all authentification code from Aodh.
Aodh become compatible with all keystone API version and
all keystone auth plugin for authentification.

This also moves the keystone project discovery to v3 API,
to fully removes the keystone v2 client from Aodh.

Implements blueprint support-keystone-v3
Change-Id: I0505616c78fc17505bdbe195a08c5ae277c6386c

aodh/api/controllers/v2/alarm_rules/gnocchi.py

@@ -63,7 +63,7 @@ class AlarmGnocchiThresholdRule(base.AlarmRule):
         ks_client = keystone_client.get_client(pecan.request.cfg)
         gnocchi_url = pecan.request.cfg.gnocchi_url
         headers = {'Content-Type': "application/json",
-                   'X-Auth-Token': ks_client.auth_token}
+                   'X-Auth-Token': keystone_client.get_auth_token(ks_client)}
         try:
             r = requests.get("%s/v1/capabilities" % gnocchi_url,
                              headers=headers)
@@ -103,7 +103,7 @@ class MetricOfResourceRule(AlarmGnocchiThresholdRule):
         ks_client = keystone_client.get_client(pecan.request.cfg)
         gnocchi_url = pecan.request.cfg.gnocchi_url
         headers = {'Content-Type': "application/json",
-                   'X-Auth-Token': ks_client.auth_token}
+                   'X-Auth-Token': keystone_client.get_auth_token(ks_client)}
         try:
             r = requests.get("%s/v1/resource/%s/%s" % (
                 gnocchi_url, rule.resource_type,
@@ -168,7 +168,8 @@ class AggregationMetricByResourcesLookupRule(AlarmGnocchiThresholdRule):
                 rule.resource_type,
                 rule.metric),
             'headers': {'Content-Type': "application/json",
-                        'X-Auth-Token': ks_client.auth_token},
+                        'X-Auth-Token': keystone_client.get_auth_token(
+                            ks_client)},
             'params': {'aggregation': rule.aggregation_method,
                        'needed_overlap': 0},
             'data': rule.query,

aodh/evaluator/gnocchi.py

@@ -20,6 +20,7 @@ import requests
 
 from aodh.evaluator import threshold
 from aodh.i18n import _
+from aodh import keystone_client
 
 LOG = log.getLogger(__name__)
 
@@ -40,7 +41,7 @@ class GnocchiThresholdEvaluator(threshold.ThresholdEvaluator):
     def _get_headers(self, content_type="application/json"):
         return {
             'Content-Type': content_type,
-            'X-Auth-Token': self.ks_client.auth_token,
+            'X-Auth-Token': keystone_client.get_auth_token(self.ks_client),
         }
 
     def _statistics(self, alarm, start, end):

aodh/evaluator/threshold.py

@@ -25,6 +25,7 @@ from oslo_utils import timeutils
 from aodh import evaluator
 from aodh.evaluator import utils
 from aodh.i18n import _, _LW
+from aodh import keystone_client
 
 LOG = log.getLogger(__name__)
 
@@ -53,20 +54,13 @@ class ThresholdEvaluator(evaluator.Evaluator):
         if self._cm_client is None:
             auth_config = self.conf.service_credentials
             self._cm_client = ceiloclient.get_client(
-                2,
+                version=2,
-                os_auth_url=auth_config.os_auth_url.replace('/v2.0', '/'),
+                session=keystone_client.get_session(self.conf),
-                os_region_name=auth_config.os_region_name,
+                # ceiloclient adapter options
-                os_tenant_name=auth_config.os_tenant_name,
+                region_name=auth_config.region_name,
-                os_password=auth_config.os_password,
+                interface=auth_config.interface,
-                os_username=auth_config.os_username,
-                os_cacert=auth_config.os_cacert,
-                os_endpoint_type=auth_config.os_endpoint_type,
-                insecure=auth_config.insecure,
-                timeout=self.conf.http_timeout,
-                os_user_domain_id=auth_config.os_user_domain_id,
-                os_project_name=auth_config.os_project_name,
-                os_project_domain_id=auth_config.os_project_domain_id,
             )
+
         return self._cm_client
 
     @classmethod

aodh/keystone_client.py

@@ -13,64 +13,60 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import os
 
-from keystoneclient import discover as ks_discover
+
-from keystoneclient import exceptions as ks_exception
+from keystoneauth1 import exceptions as ka_exception
-from keystoneclient import session as ks_session
+from keystoneauth1 import identity as ka_identity
-from keystoneclient.v2_0 import client as ks_client
+from keystoneauth1 import loading as ka_loading
 from keystoneclient.v3 import client as ks_client_v3
+from oslo_config import cfg
+from oslo_log import log
+
+LOG = log.getLogger(__name__)
+
+CFG_GROUP = "service_credentials"
 
 
-def get_client(conf):
+def get_session(conf, requests_session=None):
-    return ks_client.Client(
+    """Get a aodh service credentials auth session."""
-        username=conf.service_credentials.os_username,
+    auth_plugin = ka_loading.load_auth_from_conf_options(conf, CFG_GROUP)
-        password=conf.service_credentials.os_password,
+    session = ka_loading.load_session_from_conf_options(
-        tenant_id=conf.service_credentials.os_tenant_id,
+        conf, CFG_GROUP, auth=auth_plugin, session=requests_session
-        tenant_name=conf.service_credentials.os_tenant_name,
+    )
-        cacert=conf.service_credentials.os_cacert,
+    return session
-        auth_url=conf.service_credentials.os_auth_url,
-        region_name=conf.service_credentials.os_region_name,
-        insecure=conf.service_credentials.insecure,
-        timeout=conf.http_timeout,)
 
 
-def get_v3_client(conf, trust_id=None):
+def get_client(conf, trust_id=None, requests_session=None):
     """Return a client for keystone v3 endpoint, optionally using a trust."""
-    auth_url = conf.service_credentials.os_auth_url
+    session = get_session(conf, requests_session=requests_session)
-    try:
+    return ks_client_v3.Client(session=session, trust_id=trust_id)
-        auth_url_noneversion = auth_url.replace('/v2.0', '/')
-        discover = ks_discover.Discover(auth_url=auth_url_noneversion)
-        v3_auth_url = discover.url_for('3.0')
-        if v3_auth_url:
-            auth_url = v3_auth_url
-        else:
-            auth_url = auth_url
-    except Exception:
-        auth_url = auth_url.replace('/v2.0', '/v3')
-    return ks_client_v3.Client(
-        username=conf.service_credentials.os_username,
-        password=conf.service_credentials.os_password,
-        cacert=conf.service_credentials.os_cacert,
-        auth_url=auth_url,
-        region_name=conf.service_credentials.os_region_name,
-        insecure=conf.service_credentials.insecure,
-        timeout=conf.http_timeout,
-        trust_id=trust_id)
 
 
-def create_trust_id(conf, trustor_user_id, trustor_project_id,
+def get_service_catalog(client):
-                    roles, auth_plugin):
+    return client.session.auth.get_access(client.session).service_catalog
+
+
+def get_auth_token(client):
+    return client.session.auth.get_access(client.session).auth_token
+
+
+def get_client_on_behalf_user(conf, auth_plugin, trust_id=None,
+                              requests_session=None):
+    """Return a client for keystone v3 endpoint, optionally using a trust."""
+    session = ka_loading.load_session_from_conf_options(
+        conf, CFG_GROUP, auth=auth_plugin, session=requests_session
+    )
+    return ks_client_v3.Client(session=session, trust_id=trust_id)
+
+
+def create_trust_id(conf, trustor_user_id, trustor_project_id, roles,
+                    auth_plugin):
     """Create a new trust using the aodh service user."""
-    admin_client = get_v3_client(conf)
+    admin_client = get_client(conf)
-
     trustee_user_id = admin_client.auth_ref.user_id
 
-    session = ks_session.Session.construct({
+    client = get_client_on_behalf_user(conf, auth_plugin=auth_plugin)
-        'cacert': conf.service_credentials.os_cacert,
-        'insecure': conf.service_credentials.insecure})
-
-    client = ks_client_v3.Client(session=session, auth=auth_plugin)
-
     trust = client.trusts.create(trustor_user=trustor_user_id,
                                  trustee_user=trustee_user_id,
                                  project=trustor_project_id,
@@ -81,12 +77,94 @@ def create_trust_id(conf, trustor_user_id, trustor_project_id,
 
 def delete_trust_id(conf, trust_id, auth_plugin):
     """Delete a trust previously setup for the aodh user."""
-    session = ks_session.Session.construct({
+    client = get_client_on_behalf_user(conf, auth_plugin=auth_plugin)
-        'cacert': conf.service_credentials.os_cacert,
-        'insecure': conf.service_credentials.insecure})
-
-    client = ks_client_v3.Client(session=session, auth=auth_plugin)
     try:
         client.trusts.delete(trust_id)
-    except ks_exception.NotFound:
+    except ka_exception.NotFound:
         pass
+
+
+OPTS = [
+    cfg.StrOpt('region-name',
+               default=os.environ.get('OS_REGION_NAME'),
+               deprecated_name="os-region-name",
+               help='Region name to use for OpenStack service endpoints.'),
+    cfg.StrOpt('interface',
+               default=os.environ.get(
+                   'OS_INTERFACE', os.environ.get('OS_ENDPOINT_TYPE',
+                                                  'public')),
+               deprecated_name="os-endpoint-type",
+               choices=('public', 'internal', 'admin', 'auth', 'publicURL',
+                        'internalURL', 'adminURL'),
+               help='Type of endpoint in Identity service catalog to use for '
+                    'communication with OpenStack services.'),
+]
+
+
+def register_keystoneauth_opts(conf):
+    ka_loading.register_auth_conf_options(conf, CFG_GROUP)
+    ka_loading.register_session_conf_options(
+        conf, CFG_GROUP,
+        deprecated_opts={'cacert': [
+            cfg.DeprecatedOpt('os-cacert', group=CFG_GROUP),
+            cfg.DeprecatedOpt('os-cacert', group="DEFAULT")]
+        })
+    conf.set_default("auth_type", default="password-aodh-legacy",
+                     group=CFG_GROUP)
+
+
+def setup_keystoneauth(conf):
+    if conf[CFG_GROUP].auth_type == "password-aodh-legacy":
+        LOG.warn("Value 'password-aodh-legacy' for '[%s]/auth_type' "
+                 "is deprecated. And will be removed in Aodh 3.0. "
+                 "Use 'password' instead.",
+                 CFG_GROUP)
+    ka_loading.load_auth_from_conf_options(conf, CFG_GROUP)
+
+
+class LegacyAodhKeystoneLoader(ka_loading.BaseLoader):
+    @property
+    def plugin_class(self):
+        return ka_identity.V2Password
+
+    def get_options(self):
+        options = super(LegacyAodhKeystoneLoader, self).get_options()
+        options.extend([
+            ka_loading.Opt(
+                'os-username',
+                default=os.environ.get('OS_USERNAME', 'aodh'),
+                help='User name to use for OpenStack service access.'),
+            ka_loading.Opt(
+                'os-password',
+                secret=True,
+                default=os.environ.get('OS_PASSWORD', 'admin'),
+                help='Password to use for OpenStack service access.'),
+            ka_loading.Opt(
+                'os-tenant-id',
+                default=os.environ.get('OS_TENANT_ID', ''),
+                help='Tenant ID to use for OpenStack service access.'),
+            ka_loading.Opt(
+                'os-tenant-name',
+                default=os.environ.get('OS_TENANT_NAME', 'admin'),
+                help='Tenant name to use for OpenStack service access.'),
+            ka_loading.Opt(
+                'os-auth-url',
+                default=os.environ.get('OS_AUTH_URL',
+                                       'http://localhost:5000/v2.0'),
+                help='Auth URL to use for OpenStack service access.'),
+        ])
+        return options
+
+    def load_from_options(self, **kwargs):
+        options_map = {
+            'os_auth_url': 'auth_url',
+            'os_username': 'username',
+            'os_password': 'password',
+            'os_tenant_name': 'tenant_name',
+            'os_tenant_id': 'tenant_id',
+        }
+        identity_kwargs = dict((options_map[o.dest],
+                                kwargs.get(o.dest) or o.default)
+                               for o in self.get_options()
+                               if o.dest in options_map)
+        return self.plugin_class(**identity_kwargs)

aodh/notifier/trust.py

@@ -34,7 +34,7 @@ class TrustRestAlarmNotifier(rest.RestAlarmNotifier):
                reason, reason_data):
         trust_id = action.username
 
-        client = keystone_client.get_v3_client(self.conf, trust_id)
+        client = keystone_client.get_client(self.conf, trust_id)
 
         # Remove the fake user
         netloc = action.netloc.split("@")[1]
@@ -44,7 +44,7 @@ class TrustRestAlarmNotifier(rest.RestAlarmNotifier):
         action = parse.SplitResult(scheme, netloc, action.path, action.query,
                                    action.fragment)
 
-        headers = {'X-Auth-Token': client.auth_token}
+        headers = {'X-Auth-Token': keystone_client.get_auth_token(client)}
         super(TrustRestAlarmNotifier, self).notify(
             action, alarm_id, alarm_name, severity, previous, current, reason,
             reason_data, headers)

aodh/opts.py

@@ -13,6 +13,7 @@
 # under the License.
 import itertools
 
+from keystoneauth1 import loading
 from oslo_config import cfg
 
 import aodh.api
@@ -22,6 +23,7 @@ import aodh.evaluator
 import aodh.evaluator.event
 import aodh.evaluator.gnocchi
 import aodh.event
+import aodh.keystone_client
 import aodh.notifier.rest
 import aodh.rpc
 import aodh.service
@@ -64,5 +66,16 @@ def list_opts():
              ])),
         ('coordination', aodh.coordination.OPTS),
         ('database', aodh.storage.OPTS),
-        ('service_credentials', aodh.service.CLI_OPTS),
+        ('service_credentials', aodh.keystone_client.OPTS),
     ]
+
+
+def list_keystoneauth_opts():
+    # NOTE(sileht): the configuration file contains only the options
+    # for the password plugin that handles keystone v2 and v3 API
+    # with discovery. But other options are possible.
+    # Also, the default loaded plugin is password-aodh-legacy for
+    # backward compatibily
+    return [('service_credentials', (
+            loading.get_auth_common_conf_options() +
+            loading.get_auth_plugin_conf_options('password')))]

aodh/service.py

@@ -14,7 +14,6 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
-import os
 import socket
 
 from oslo_config import cfg
@@ -23,6 +22,7 @@ import oslo_i18n
 from oslo_log import log
 from oslo_policy import opts as policy_opts
 
+from aodh import keystone_client
 from aodh import messaging
 
 
@@ -47,50 +47,6 @@ OPTS = [
 ]
 
 
-CLI_OPTS = [
-    cfg.StrOpt('os-username',
-               default=os.environ.get('OS_USERNAME', 'aodh'),
-               help='User name to use for OpenStack service access.'),
-    cfg.StrOpt('os-password',
-               secret=True,
-               default=os.environ.get('OS_PASSWORD', 'admin'),
-               help='Password to use for OpenStack service access.'),
-    cfg.StrOpt('os-tenant-id',
-               default=os.environ.get('OS_TENANT_ID', ''),
-               help='Tenant ID to use for OpenStack service access.'),
-    cfg.StrOpt('os-tenant-name',
-               default=os.environ.get('OS_TENANT_NAME', 'admin'),
-               help='Tenant name to use for OpenStack service access.'),
-    cfg.StrOpt('os-cacert',
-               default=os.environ.get('OS_CACERT'),
-               help='Certificate chain for SSL validation.'),
-    cfg.StrOpt('os-auth-url',
-               default=os.environ.get('OS_AUTH_URL',
-                                      'http://localhost:5000/v2.0'),
-               help='Auth URL to use for OpenStack service access.'),
-    cfg.StrOpt('os-region-name',
-               default=os.environ.get('OS_REGION_NAME'),
-               help='Region name to use for OpenStack service endpoints.'),
-    cfg.StrOpt('os-endpoint-type',
-               default=os.environ.get('OS_ENDPOINT_TYPE', 'publicURL'),
-               help='Type of endpoint in Identity service catalog to use for '
-                    'communication with OpenStack services.'),
-    cfg.BoolOpt('insecure',
-                default=False,
-                help='Disables X.509 certificate validation when an '
-                     'SSL connection to Identity Service is established.'),
-    cfg.StrOpt('os-user-domain-id',
-               default=os.environ.get('OS_USER_DOMAIN_ID', 'default'),
-               help='The domain id of the user'),
-    cfg.StrOpt('os-project-domain-id',
-               default=os.environ.get('OS_PROJECT_DOMAIN_ID', 'default'),
-               help='The domain id of the user project'),
-    cfg.StrOpt('os-project-name',
-               default=os.environ.get('OS_PROJECT_NAME', 'admin'),
-               help='The user project name'),
-]
-
-
 def prepare_service(argv=None, config_files=None):
     conf = cfg.ConfigOpts()
     oslo_i18n.enable_lazy()
@@ -105,9 +61,12 @@ def prepare_service(argv=None, config_files=None):
     for group, options in opts.list_opts():
         conf.register_opts(list(options),
                            group=None if group == "DEFAULT" else group)
+    keystone_client.register_keystoneauth_opts(conf)
 
     conf(argv, project='aodh', validate_default_values=True,
          default_config_files=config_files)
+
+    keystone_client.setup_keystoneauth(conf)
     log.setup(conf, 'aodh')
     messaging.setup()
     return conf

aodh/tests/functional/api/v2/test_alarm_scenarios.py

@@ -1320,7 +1320,7 @@ class TestAlarms(TestAlarmsBase):
         }
         auth = mock.Mock()
         trust_client = mock.Mock()
-        with mock.patch('aodh.keystone_client.get_v3_client') as client:
+        with mock.patch('aodh.keystone_client.get_client') as client:
             client.return_value = mock.Mock(
                 auth_ref=mock.Mock(user_id='my_user'))
             with mock.patch('keystoneclient.v3.client.Client') as sub_client:
@@ -1345,7 +1345,7 @@ class TestAlarms(TestAlarmsBase):
         else:
             self.fail("Alarm not found")
 
-        with mock.patch('aodh.keystone_client.get_v3_client') as client:
+        with mock.patch('aodh.keystone_client.get_client') as client:
             client.return_value = mock.Mock(
                 auth_ref=mock.Mock(user_id='my_user'))
             with mock.patch('keystoneclient.v3.client.Client') as sub_client:
@@ -1571,7 +1571,7 @@ class TestAlarms(TestAlarmsBase):
         data = self._get_alarm('a')
         data.update({'ok_actions': ['trust+http://something/ok']})
         trust_client = mock.Mock()
-        with mock.patch('aodh.keystone_client.get_v3_client') as client:
+        with mock.patch('aodh.keystone_client.get_client') as client:
             client.return_value = mock.Mock(
                 auth_ref=mock.Mock(user_id='my_user'))
             with mock.patch('keystoneclient.v3.client.Client') as sub_client:
@@ -1586,7 +1586,7 @@ class TestAlarms(TestAlarmsBase):
 
         data.update({'ok_actions': ['http://no-trust-something/ok']})
 
-        with mock.patch('aodh.keystone_client.get_v3_client') as client:
+        with mock.patch('aodh.keystone_client.get_client') as client:
             client.return_value = mock.Mock(
                 auth_ref=mock.Mock(user_id='my_user'))
             with mock.patch('keystoneclient.v3.client.Client') as sub_client:

aodh/tests/unit/evaluator/test_gnocchi.py

@@ -198,7 +198,7 @@ class TestGnocchiThresholdEvaluate(base.TestEvaluatorBase):
         self.requests.post.side_effect = [avgs2]
         self._evaluate_all_alarms()
 
-        expected_headers = {'X-Auth-Token': 'fake_token',
+        expected_headers = {'X-Auth-Token': mock.ANY,
                             'Content-Type': 'application/json'}
 
         start_alarm1 = "2015-01-26T12:51:00"

aodh/tests/unit/test_notifier.py

@@ -243,7 +243,7 @@ class TestAlarmNotifier(tests_base.BaseTestCase):
         url = 'http://host/action'
 
         client = mock.MagicMock()
-        client.auth_token = 'token_1234'
+        client.session.auth.get_access.return_value.auth_token = 'token_1234'
         headers = {'X-Auth-Token': 'token_1234'}
         headers.update(self.HTTP_HEADERS)
 

etc/aodh/aodh-config-generator.conf

@@ -2,6 +2,7 @@
 output_file = etc/aodh/aodh.conf
 wrap_width = 79
 namespace = aodh
+namespace = aodh-auth
 namespace = oslo.db
 namespace = oslo.log
 namespace = oslo.messaging

setup.cfg

@@ -106,6 +106,10 @@ console_scripts =
 
 oslo.config.opts =
     aodh = aodh.opts:list_opts
+    aodh-auth = aodh.opts:list_keystoneauth_opts
+
+keystoneauth1.plugin =
+    password-aodh-legacy = aodh.keystone_client:LegacyAodhKeystoneLoader
 
 [build_sphinx]
 all_files = 1