Work with OpenStack python client in a modern manner

* Replace all home-brewn code to work with OpenStack clients
  with os-client-config lib. No need to monitor token expiration
  for heat client anymore!
* Add 'os-project-name' parameter as it becomes standard
* Sync requirements to the latest

Closes-Bug: 1573504

Change-Id: I7520b9aed075074b4b47551eb22d18e568da83dd

doc/source/tools/shaker-cleanup.txt

@@ -10,6 +10,7 @@ usage: shaker-cleanup [-h] [--cleanup-on-error] [--config-dir DIR]
                       [--nowatch-log-file] [--os-auth-url <auth-url>]
                       [--os-cacert <auth-cacert>] [--os-insecure]
                       [--os-password <auth-password>]
+                      [--os-project-name <auth-project-name>]
                       [--os-region-name <auth-region-name>]
                       [--os-tenant-name <auth-tenant-name>]
                       [--os-username <auth-username>]
@@ -84,6 +85,10 @@ optional arguments:
                         defaults to env[OS_INSECURE].
   --os-password <auth-password>
                         Authentication password, defaults to env[OS_PASSWORD].
+  --os-project-name <auth-project-name>
+                        Another way to specify tenant name. This option is
+                        mutually exclusive with --os-tenant-name. Defaults to
+                        env[OS_PROJECT_NAME].
   --os-region-name <auth-region-name>
                         Authentication region name, defaults to
                         env[OS_REGION_NAME].

doc/source/tools/shaker-image-builder.txt

@@ -12,6 +12,7 @@ usage: shaker-image-builder [-h] [--cleanup-on-error] [--config-dir DIR]
                             [--nowatch-log-file] [--os-auth-url <auth-url>]
                             [--os-cacert <auth-cacert>] [--os-insecure]
                             [--os-password <auth-password>]
+                            [--os-project-name <auth-project-name>]
                             [--os-region-name <auth-region-name>]
                             [--os-tenant-name <auth-tenant-name>]
                             [--os-username <auth-username>]
@@ -86,6 +87,10 @@ optional arguments:
                         defaults to env[OS_INSECURE].
   --os-password <auth-password>
                         Authentication password, defaults to env[OS_PASSWORD].
+  --os-project-name <auth-project-name>
+                        Another way to specify tenant name. This option is
+                        mutually exclusive with --os-tenant-name. Defaults to
+                        env[OS_PROJECT_NAME].
   --os-region-name <auth-region-name>
                         Authentication region name, defaults to
                         env[OS_REGION_NAME].

doc/source/tools/shaker.txt

@@ -10,6 +10,7 @@ usage: shaker [-h] [--agent-join-timeout AGENT_JOIN_TIMEOUT]
               [--noverbose] [--nowatch-log-file] [--os-auth-url <auth-url>]
               [--os-cacert <auth-cacert>] [--os-insecure]
               [--os-password <auth-password>]
+              [--os-project-name <auth-project-name>]
               [--os-region-name <auth-region-name>]
               [--os-tenant-name <auth-tenant-name>]
               [--os-username <auth-username>] [--output OUTPUT]
@@ -99,6 +100,10 @@ optional arguments:
                         defaults to env[OS_INSECURE].
   --os-password <auth-password>
                         Authentication password, defaults to env[OS_PASSWORD].
+  --os-project-name <auth-project-name>
+                        Another way to specify tenant name. This option is
+                        mutually exclusive with --os-tenant-name. Defaults to
+                        env[OS_PROJECT_NAME].
   --os-region-name <auth-region-name>
                         Authentication region name, defaults to
                         env[OS_REGION_NAME].

etc/shaker.conf

@@ -112,6 +112,10 @@
 # Authentication tenant name, defaults to env[OS_TENANT_NAME]. (string value)
 #os_tenant_name =
 
+# Another way to specify tenant name. This option is mutually exclusive with
+# --os-tenant-name. Defaults to env[OS_PROJECT_NAME]. (string value)
+#os_project_name =
+
 # Authentication username, defaults to env[OS_USERNAME]. (string value)
 #os_username =
 

requirements.txt

@@ -2,25 +2,26 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-pbr>=1.6
+pbr>=1.6 # Apache-2.0
 
-iso8601>=0.1.9
+iso8601>=0.1.9 # MIT
-Jinja2>=2.6 # BSD License (3 clause)
+Jinja2>=2.8 # BSD License (3 clause)
-oslo.concurrency>=2.3.0 # Apache-2.0
+keystoneauth1>=2.1.0 # Apache-2.0
-oslo.config>=2.6.0 # Apache-2.0
+os-client-config>=1.13.1 # Apache-2.0
-oslo.i18n>=1.5.0 # Apache-2.0
+oslo.concurrency>=3.5.0 # Apache-2.0
-oslo.log>=1.12.0 # Apache-2.0
+oslo.config>=3.9.0 # Apache-2.0
+oslo.i18n>=2.1.0 # Apache-2.0
+oslo.log>=1.14.0 # Apache-2.0
 oslo.serialization>=1.10.0 # Apache-2.0
-oslo.utils!=2.6.0,>=2.4.0 # Apache-2.0
+oslo.utils>=3.5.0 # Apache-2.0
-psutil<2.0.0,>=1.1.1
+psutil<2.0.0,>=1.1.1 # BSD
 pygal
 pykwalify
-python-glanceclient>=0.18.0
+python-glanceclient>=2.0.0 # Apache-2.0
-python-keystoneclient!=1.8.0,>=1.6.0
+python-neutronclient>=4.2.0 # Apache-2.0
-python-neutronclient>=2.6.0
+python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
-python-novaclient!=2.33.0,>=2.29.0
+python-heatclient>=0.6.0 # Apache-2.0
-python-heatclient>=0.6.0
+python-subunit>=0.0.18 # Apache-2.0/BSD
-python-subunit>=0.0.18
+PyYAML>=3.1.0 # MIT
-PyYAML>=3.1.0
 pyzmq>=15.2.0
-six>=1.9.0
+six>=1.9.0 # MIT

shaker/engine/config.py

@@ -78,6 +78,12 @@ OPENSTACK_OPTS = [
                sample_default='',
                help='Authentication tenant name, defaults to '
                     'env[OS_TENANT_NAME].'),
+    cfg.StrOpt('os-project-name', metavar='<auth-project-name>',
+               default=utils.env('OS_PROJECT_NAME'),
+               sample_default='',
+               help='Another way to specify tenant name. This option is '
+                    'mutually exclusive with --os-tenant-name. '
+                    'Defaults to env[OS_PROJECT_NAME].'),
     cfg.StrOpt('os-username', metavar='<auth-username>',
                default=utils.env('OS_USERNAME'),
                sample_default='',

shaker/engine/deploy.py

@@ -220,15 +220,11 @@ class Deployment(object):
         self.has_stack = False
         self.privileged_mode = True
 
-    def connect_to_openstack(self, os_username, os_password, os_tenant_name,
+    def connect_to_openstack(self, openstack_params, flavor_name, image_name,
-                             os_auth_url, os_region_name, external_net,
+                             external_net):
-                             flavor_name, image_name, os_cacert, os_insecure):
         LOG.debug('Connecting to OpenStack')
 
-        self.openstack_client = openstack.OpenStackClient(
+        self.openstack_client = openstack.OpenStackClient(openstack_params)
-            username=os_username, password=os_password,
-            tenant_name=os_tenant_name, auth_url=os_auth_url,
-            region_name=os_region_name, cacert=os_cacert, insecure=os_insecure)
 
         self.flavor_name = flavor_name
         self.image_name = image_name

shaker/engine/image_builder.py

@@ -34,23 +34,14 @@ def init():
     utils.init_config_and_logging(
         config.OPENSTACK_OPTS + config.IMAGE_BUILDER_OPTS)
 
-    openstack_client = None
+    openstack_params = utils.pack_openstack_params(cfg.CONF)
     try:
-        openstack_client = openstack.OpenStackClient(
+        return openstack.OpenStackClient(openstack_params)
-            username=cfg.CONF.os_username, password=cfg.CONF.os_password,
-            tenant_name=cfg.CONF.os_tenant_name, auth_url=cfg.CONF.os_auth_url,
-            region_name=cfg.CONF.os_region_name, cacert=cfg.CONF.os_cacert,
-            insecure=cfg.CONF.os_insecure
-        )
     except Exception as e:
-        LOG.error('Error establishing connection to OpenStack: %s. '
+        LOG.error('Failed to connect to OpenStack: %s. '
-                  'Please verify OpenStack credentials (--os-username, '
+                  'Please verify parameters: %s', e, openstack_params)
-                  '--os-password, --os-tenant-name, --os-auth-url, '
-                  '--os-cacert, --os-insecure)', e)
         exit(1)
 
-    return openstack_client
-
 
 def build_image():
     openstack_client = init()

shaker/engine/server.py

@@ -136,7 +136,7 @@ def execute(output, quorum, execution, agents, matrix=None):
 def _under_openstack():
     required = ['os_username', 'os_password', 'os_tenant_name', 'os_auth_url']
     for param in required:
-        if param not in cfg.CONF or not cfg.CONF[param]:
+        if param not in cfg.CONF:
             return False
     return True
 
@@ -149,12 +149,14 @@ def play_scenario(scenario):
         deployment = deploy.Deployment()
 
         if _under_openstack():
-            deployment.connect_to_openstack(
+            openstack_params = utils.pack_openstack_params(cfg.CONF)
-                cfg.CONF.os_username, cfg.CONF.os_password,
+            try:
-                cfg.CONF.os_tenant_name, cfg.CONF.os_auth_url,
+                deployment.connect_to_openstack(
-                cfg.CONF.os_region_name, cfg.CONF.external_net,
+                    openstack_params, cfg.CONF.flavor_name,
-                cfg.CONF.flavor_name, cfg.CONF.image_name,
+                    cfg.CONF.image_name, cfg.CONF.external_net)
-                cfg.CONF.os_cacert, cfg.CONF.os_insecure)
+            except Exception as e:
+                LOG.warning('Failed to connect to OpenStack: %s. Please '
+                            'verify parameters: %s', e, openstack_params)
 
         base_dir = os.path.dirname(scenario['file_name'])
         scenario_deployment = scenario.get('deployment', {})
@@ -192,8 +194,7 @@ def play_scenario(scenario):
             record = dict(id=utils.make_record_id(), status='interrupted')
         else:
             error_msg = 'Error while executing scenario: %s' % e
-            LOG.error(error_msg)
+            LOG.error(error_msg, exc_info=True)
-            LOG.exception(e)
             record = dict(id=utils.make_record_id(), status='error',
                           stderr=error_msg)
         output['records'][record['id']] = record

shaker/engine/utils.py

@@ -57,8 +57,12 @@ def init_config_and_logging(opts):
     conf.register_cli_opts(opts)
     conf.register_opts(opts)
     logging.register_options(conf)
-    logging.set_defaults(
+
-        default_log_levels=conf.default_log_levels + ['pykwalify=INFO'])
+    # requests to OpenStack services should be visible at DEBUG level
+    default_log_levels = [l for l in conf.default_log_levels
+                          if not l.startswith('keystoneauth')]
+    default_log_levels += ['pykwalify=INFO']
+    logging.set_defaults(default_log_levels=default_log_levels)
 
     try:
         conf(project='shaker')
@@ -250,3 +254,17 @@ def copy_value_by_path(src, src_param, dst, dst_param):
         set_value_by_path(dst, dst_param, v)
         return True
     return False
+
+
+def pack_openstack_params(conf):
+    params = dict(auth=dict(username=cfg.CONF.os_username,
+                            password=cfg.CONF.os_password,
+                            auth_url=cfg.CONF.os_auth_url),
+                  os_region_name=cfg.CONF.os_region_name,
+                  os_cacert=cfg.CONF.os_cacert,
+                  os_insecure=cfg.CONF.os_insecure)
+    if cfg.CONF.os_tenant_name:
+        params['auth']['tenant_name'] = cfg.CONF.os_tenant_name
+    if cfg.CONF.os_project_name:
+        params['auth']['project_name'] = cfg.CONF.os_project_name
+    return params

shaker/openstack/clients/glance.py

@@ -13,21 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from glanceclient import client as glance_client_pkg
-
-
-GLANCE_VERSION = '1'
-
-
-def create_client(keystone_client, os_region_name, cacert, insecure):
-    image_api_url = keystone_client.service_catalog.url_for(
-        service_type='image', region_name=os_region_name)
-    return glance_client_pkg.Client(GLANCE_VERSION,
-                                    endpoint=image_api_url,
-                                    token=keystone_client.auth_token,
-                                    cacert=cacert,
-                                    insecure=insecure)
-
 
 def get_image(glance_client, image_name):
     for image in glance_client.images.list():

shaker/openstack/clients/heat.py

@@ -15,26 +15,12 @@
 
 import time
 
-from heatclient import client as heat_client_pkg
 from oslo_log import log as logging
 
 
 LOG = logging.getLogger(__name__)
 
 
-HEAT_VERSION = '1'
-
-
-def create_client(keystone_client, os_region_name, cacert, insecure):
-    orchestration_api_url = keystone_client.service_catalog.url_for(
-        service_type='orchestration', region_name=os_region_name)
-    return heat_client_pkg.Client(HEAT_VERSION,
-                                  endpoint=orchestration_api_url,
-                                  token=keystone_client.auth_token,
-                                  ca_file=cacert,
-                                  insecure=insecure)
-
-
 def create_stack(heat_client, stack_name, template, parameters):
     stack_params = {
         'stack_name': stack_name,

shaker/openstack/clients/keystone.py

@@ -1,37 +0,0 @@
-# Copyright (c) 2015 Mirantis Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from keystoneclient.auth.identity import v2 as auth_v2
-from keystoneclient import discover as keystone_discover
-from keystoneclient import session
-from keystoneclient.v2_0 import client as keystone_v2
-from keystoneclient.v3 import client as keystone_v3
-
-
-def create_keystone_client(**kwargs):
-    discover = keystone_discover.Discover(**kwargs)
-    for version_data in discover.version_data():
-        version = version_data["version"]
-        if version[0] <= 2:
-            return keystone_v2.Client(**kwargs)
-        elif version[0] == 3:
-            return keystone_v3.Client(**kwargs)
-    raise Exception(
-        'Failed to discover keystone version for url %(auth_url)s.', **kwargs)
-
-
-def create_keystone_session(cacert, insecure, **kwargs):
-    auth = auth_v2.Password(**kwargs)
-    return session.Session(auth=auth, cert=cacert, verify=not insecure)

shaker/openstack/clients/neutron.py

@@ -13,22 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from neutronclient.neutron import client as neutron_client_pkg
 from oslo_log import log as logging
 
 
 LOG = logging.getLogger(__name__)
 
 
-NEUTRON_VERSION = '2.0'
-
-
-def create_client(keystone_session, os_region_name):
-    return neutron_client_pkg.Client(NEUTRON_VERSION,
-                                     session=keystone_session,
-                                     region_name=os_region_name)
-
-
 def choose_external_net(neutron_client):
     ext_nets = neutron_client.list_networks(
         **{'router:external': True})['networks']

shaker/openstack/clients/nova.py

@@ -23,18 +23,11 @@ from oslo_log import log as logging
 
 LOG = logging.getLogger(__name__)
 
-NOVA_VERSION = '2'
-
 
 class ForbiddenException(nova_client_pkg.exceptions.Forbidden):
     pass
 
 
-def create_client(keystone_session, os_region_name):
-    return nova_client_pkg.Client(NOVA_VERSION, session=keystone_session,
-                                  region_name=os_region_name)
-
-
 def get_available_compute_nodes(nova_client):
     try:
         return [dict(host=svc.host, zone=svc.zone)

shaker/openstack/clients/openstack.py

@@ -13,90 +13,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import functools
+import os_client_config
-import time
+from oslo_log import log as logging
-
-from shaker.openstack.clients import glance
-from shaker.openstack.clients import heat
-from shaker.openstack.clients import keystone
-from shaker.openstack.clients import neutron
-from shaker.openstack.clients import nova
 
 
-# As of now only Nova and Neutron clients support Keystone sessions.
+LOG = logging.getLogger(__name__)
-# Thus the only way to create clients is from keystone client instance
-# and auth token. The token gets expired in an hour and there is no
-# way to automatically refresh it. So the current implementation is to
-# recreate keystone client from scratch
-
-MODERN_CLIENT_MAKERS = {
-    'neutron': neutron.create_client,
-    'nova': nova.create_client,
-}
-OLD_CLIENT_MAKERS = {
-    'glance': glance.create_client,
-    'heat': heat.create_client,
-}
-
-KEYSTONE_AUTH_EXPIRATION = 60
-
-
-class OpenStackClientProxy(object):
-    def __init__(self, keystone_creator, client_creator):
-        self.keystone_creator = keystone_creator
-        self.client_creator = client_creator
-        self.last_update_time = 0
-
-    def __getattribute__(self, name):
-        if name in ['keystone_creator', 'client_creator',
-                    'client', 'last_update_time']:
-            return super(OpenStackClientProxy, self).__getattribute__(name)
-        else:
-            now = int(time.time())
-            if now > self.last_update_time + KEYSTONE_AUTH_EXPIRATION:
-                self.last_update_time = now
-                self.client = self.client_creator(
-                    keystone_client=self.keystone_creator())
-            return self.client.__getattribute__(name)
 
 
 class OpenStackClient(object):
-    def __init__(self, username, password, tenant_name, auth_url, region_name,
+    def __init__(self, openstack_params):
-                 cacert, insecure):
+        LOG.debug('Establishing connection to OpenStack')
-        self.region_name = region_name or 'RegionOne'
-        self.cacert = cacert or ''
-        self.insecure = insecure or False
-        self._osc_cache = {}
-        self.keystone_creator = functools.partial(
-            keystone.create_keystone_client,
-            username=username, password=password,
-            tenant_name=tenant_name, auth_url=auth_url, cacert=cacert,
-            insecure=insecure)
-        self.session_creator = functools.partial(
-            keystone.create_keystone_session, cacert,
-            username=username, password=password,
-            tenant_name=tenant_name, auth_url=auth_url,
-            insecure=insecure)
-        # ping OpenStack
-        self.keystone_creator()
 
-    def __getattribute__(self, name):
+        config = os_client_config.OpenStackConfig()
-        if name != '_osc_cache' and name in self._osc_cache:
+        cloud_config = config.get_one_cloud(**openstack_params)
-            return self._osc_cache[name]
 
-        client = None
+        self.keystone_session = cloud_config.get_session()
-        if name in MODERN_CLIENT_MAKERS:
+        self.nova = cloud_config.get_legacy_client('compute')
-            session = self.session_creator()
+        self.neutron = cloud_config.get_legacy_client('network')
-            client = MODERN_CLIENT_MAKERS[name](session, self.region_name)
+        self.glance = cloud_config.get_legacy_client('image')
-        elif name in OLD_CLIENT_MAKERS:
+        self.heat = cloud_config.get_legacy_client('orchestration')
-            client_creator = functools.partial(
-                OLD_CLIENT_MAKERS[name], os_region_name=self.region_name,
-                cacert=self.cacert, insecure=self.insecure)
-            client = OpenStackClientProxy(self.keystone_creator,
-                                          client_creator)
 
-        if client:
+        # Ping OpenStack
-            self._osc_cache[name] = client
+        self.keystone_session.get_token()
-            return client
+
-        else:
+        LOG.info('Connection to OpenStack is initialized')
-            return super(OpenStackClient, self).__getattribute__(name)

shaker/tests/test_server.py

@@ -184,10 +184,13 @@ class TestServerPlayScenario(testtools.TestCase):
         deploy_obj.deploy.assert_called_once_with(
             self.deployment, base_dir='folder',
             server_endpoint='127.0.0.1:5999')
+        openstack_params = dict(
+            auth=dict(username='user', password='password',
+                      tenant_name='tenant', auth_url='auth-url'),
+            os_region_name='RegionOne',
+            os_cacert=None, os_insecure=False)
         deploy_obj.connect_to_openstack.assert_called_once_with(
-            'user', 'password', 'tenant', 'auth-url', 'RegionOne', None,
+            openstack_params, 'shaker-flavor', 'shaker-image', None)
-            'shaker-flavor', 'shaker-image', None, False
-        )
         deploy_obj.cleanup.assert_called_once_with()
 
     @mock.patch('shaker.engine.deploy.Deployment')

test-requirements.txt

@@ -3,12 +3,12 @@
 # process, which may cause wedges in the gate later.
 
 # Hacking already pins down pep8, pyflakes and flake8
-coverage>=3.6
+coverage>=3.6 # Apache-2.0
 hacking<0.11,>=0.10
-mock>=1.2
+mock>=1.2 # BSD
 oslotest>=1.10.0 # Apache-2.0
-sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
+sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
-sphinxcontrib-httpdomain
+sphinxcontrib-httpdomain # BSD
 sphinx_rtd_theme
-testrepository>=0.0.18
+testrepository>=0.0.18 # Apache-2.0/BSD
-testtools>=1.4.0
+testtools>=1.4.0 # MIT