performa/shaker
Code Issues Proposed changes

Work with OpenStack python client in a modern manner

Browse Source 
* Replace all home-brewn code to work with OpenStack clients
  with os-client-config lib. No need to monitor token expiration
  for heat client anymore!
* Add 'os-project-name' parameter as it becomes standard
* Sync requirements to the latest

Closes-Bug: 1573504

Change-Id: I7520b9aed075074b4b47551eb22d18e568da83dd
This commit is contained in:
Ilya Shakhat 2016-04-22 17:40:23 +03:00
parent 51b9ca6393
commit 46216f0ad1
18 changed files with 108 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
doc/source/tools/shaker-cleanup.txt
View File

@ -10,6 +10,7 @@ usage: shaker-cleanup [-h] [--cleanup-on-error] [--config-dir DIR]
[--nowatch-log-file] [--os-auth-url <auth-url>]
[--os-cacert <auth-cacert>] [--os-insecure]
[--os-password <auth-password>]
[--os-project-name <auth-project-name>]
[--os-region-name <auth-region-name>]
[--os-tenant-name <auth-tenant-name>]
[--os-username <auth-username>]
@ -84,6 +85,10 @@ optional arguments:
defaults to env[OS_INSECURE].
--os-password <auth-password>
Authentication password, defaults to env[OS_PASSWORD].
--os-project-name <auth-project-name>
Another way to specify tenant name. This option is
mutually exclusive with --os-tenant-name. Defaults to
env[OS_PROJECT_NAME].
--os-region-name <auth-region-name>
Authentication region name, defaults to
env[OS_REGION_NAME].

5
doc/source/tools/shaker-image-builder.txt
View File

@ -12,6 +12,7 @@ usage: shaker-image-builder [-h] [--cleanup-on-error] [--config-dir DIR]
[--nowatch-log-file] [--os-auth-url <auth-url>]
[--os-cacert <auth-cacert>] [--os-insecure]
[--os-password <auth-password>]
[--os-project-name <auth-project-name>]
[--os-region-name <auth-region-name>]
[--os-tenant-name <auth-tenant-name>]
[--os-username <auth-username>]
@ -86,6 +87,10 @@ optional arguments:
defaults to env[OS_INSECURE].
--os-password <auth-password>
Authentication password, defaults to env[OS_PASSWORD].
--os-project-name <auth-project-name>
Another way to specify tenant name. This option is
mutually exclusive with --os-tenant-name. Defaults to
env[OS_PROJECT_NAME].
--os-region-name <auth-region-name>
Authentication region name, defaults to
env[OS_REGION_NAME].

5
doc/source/tools/shaker.txt
View File

@ -10,6 +10,7 @@ usage: shaker [-h] [--agent-join-timeout AGENT_JOIN_TIMEOUT]
[--noverbose] [--nowatch-log-file] [--os-auth-url <auth-url>]
[--os-cacert <auth-cacert>] [--os-insecure]
[--os-password <auth-password>]
[--os-project-name <auth-project-name>]
[--os-region-name <auth-region-name>]
[--os-tenant-name <auth-tenant-name>]
[--os-username <auth-username>] [--output OUTPUT]
@ -99,6 +100,10 @@ optional arguments:
defaults to env[OS_INSECURE].
--os-password <auth-password>
Authentication password, defaults to env[OS_PASSWORD].
--os-project-name <auth-project-name>
Another way to specify tenant name. This option is
mutually exclusive with --os-tenant-name. Defaults to
env[OS_PROJECT_NAME].
--os-region-name <auth-region-name>
Authentication region name, defaults to
env[OS_REGION_NAME].

4
etc/shaker.conf
View File

@ -112,6 +112,10 @@
# Authentication tenant name, defaults to env[OS_TENANT_NAME]. (string value)
#os_tenant_name =
# Another way to specify tenant name. This option is mutually exclusive with
# --os-tenant-name. Defaults to env[OS_PROJECT_NAME]. (string value)
#os_project_name =
# Authentication username, defaults to env[OS_USERNAME]. (string value)
#os_username =

35
requirements.txt
View File

@ -2,25 +2,26 @@
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
pbr>=1.6
pbr>=1.6 # Apache-2.0
iso8601>=0.1.9
Jinja2>=2.6 # BSD License (3 clause)
oslo.concurrency>=2.3.0 # Apache-2.0
oslo.config>=2.6.0 # Apache-2.0
oslo.i18n>=1.5.0 # Apache-2.0
oslo.log>=1.12.0 # Apache-2.0
iso8601>=0.1.9 # MIT
Jinja2>=2.8 # BSD License (3 clause)
keystoneauth1>=2.1.0 # Apache-2.0
os-client-config>=1.13.1 # Apache-2.0
oslo.concurrency>=3.5.0 # Apache-2.0
oslo.config>=3.9.0 # Apache-2.0
oslo.i18n>=2.1.0 # Apache-2.0
oslo.log>=1.14.0 # Apache-2.0
oslo.serialization>=1.10.0 # Apache-2.0
oslo.utils!=2.6.0,>=2.4.0 # Apache-2.0
psutil<2.0.0,>=1.1.1
oslo.utils>=3.5.0 # Apache-2.0
psutil<2.0.0,>=1.1.1 # BSD
pygal
pykwalify
python-glanceclient>=0.18.0
python-keystoneclient!=1.8.0,>=1.6.0
python-neutronclient>=2.6.0
python-novaclient!=2.33.0,>=2.29.0
python-heatclient>=0.6.0
python-subunit>=0.0.18
PyYAML>=3.1.0
python-glanceclient>=2.0.0 # Apache-2.0
python-neutronclient>=4.2.0 # Apache-2.0
python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
python-heatclient>=0.6.0 # Apache-2.0
python-subunit>=0.0.18 # Apache-2.0/BSD
PyYAML>=3.1.0 # MIT
pyzmq>=15.2.0
six>=1.9.0
six>=1.9.0 # MIT

6
shaker/engine/config.py
View File

@ -78,6 +78,12 @@ OPENSTACK_OPTS = [
sample_default='',
help='Authentication tenant name, defaults to '
'env[OS_TENANT_NAME].'),
cfg.StrOpt('os-project-name', metavar='<auth-project-name>',
default=utils.env('OS_PROJECT_NAME'),
sample_default='',
help='Another way to specify tenant name. This option is '
'mutually exclusive with --os-tenant-name. '
'Defaults to env[OS_PROJECT_NAME].'),
cfg.StrOpt('os-username', metavar='<auth-username>',
default=utils.env('OS_USERNAME'),
sample_default='',

10
shaker/engine/deploy.py
View File

@ -220,15 +220,11 @@ class Deployment(object):
self.has_stack = False
self.privileged_mode = True
def connect_to_openstack(self, os_username, os_password, os_tenant_name,
os_auth_url, os_region_name, external_net,
flavor_name, image_name, os_cacert, os_insecure):
def connect_to_openstack(self, openstack_params, flavor_name, image_name,
external_net):
LOG.debug('Connecting to OpenStack')
self.openstack_client = openstack.OpenStackClient(
username=os_username, password=os_password,
tenant_name=os_tenant_name, auth_url=os_auth_url,
region_name=os_region_name, cacert=os_cacert, insecure=os_insecure)
self.openstack_client = openstack.OpenStackClient(openstack_params)
self.flavor_name = flavor_name
self.image_name = image_name

17
shaker/engine/image_builder.py
View File

@ -34,23 +34,14 @@ def init():
utils.init_config_and_logging(
config.OPENSTACK_OPTS + config.IMAGE_BUILDER_OPTS)
openstack_client = None
openstack_params = utils.pack_openstack_params(cfg.CONF)
try:
openstack_client = openstack.OpenStackClient(
username=cfg.CONF.os_username, password=cfg.CONF.os_password,
tenant_name=cfg.CONF.os_tenant_name, auth_url=cfg.CONF.os_auth_url,
region_name=cfg.CONF.os_region_name, cacert=cfg.CONF.os_cacert,
insecure=cfg.CONF.os_insecure
)
return openstack.OpenStackClient(openstack_params)
except Exception as e:
LOG.error('Error establishing connection to OpenStack: %s. '
'Please verify OpenStack credentials (--os-username, '
'--os-password, --os-tenant-name, --os-auth-url, '
'--os-cacert, --os-insecure)', e)
LOG.error('Failed to connect to OpenStack: %s. '
'Please verify parameters: %s', e, openstack_params)
exit(1)
return openstack_client
def build_image():
openstack_client = init()

19
shaker/engine/server.py
View File

@ -136,7 +136,7 @@ def execute(output, quorum, execution, agents, matrix=None):
def _under_openstack():
required = ['os_username', 'os_password', 'os_tenant_name', 'os_auth_url']
for param in required:
if param not in cfg.CONF or not cfg.CONF[param]:
if param not in cfg.CONF:
return False
return True
@ -149,12 +149,14 @@ def play_scenario(scenario):
deployment = deploy.Deployment()
if _under_openstack():
deployment.connect_to_openstack(
cfg.CONF.os_username, cfg.CONF.os_password,
cfg.CONF.os_tenant_name, cfg.CONF.os_auth_url,
cfg.CONF.os_region_name, cfg.CONF.external_net,
cfg.CONF.flavor_name, cfg.CONF.image_name,
cfg.CONF.os_cacert, cfg.CONF.os_insecure)
openstack_params = utils.pack_openstack_params(cfg.CONF)
try:
deployment.connect_to_openstack(
openstack_params, cfg.CONF.flavor_name,
cfg.CONF.image_name, cfg.CONF.external_net)
except Exception as e:
LOG.warning('Failed to connect to OpenStack: %s. Please '
'verify parameters: %s', e, openstack_params)
base_dir = os.path.dirname(scenario['file_name'])
scenario_deployment = scenario.get('deployment', {})
@ -192,8 +194,7 @@ def play_scenario(scenario):
record = dict(id=utils.make_record_id(), status='interrupted')
else:
error_msg = 'Error while executing scenario: %s' % e
LOG.error(error_msg)
LOG.exception(e)
LOG.error(error_msg, exc_info=True)
record = dict(id=utils.make_record_id(), status='error',
stderr=error_msg)
output['records'][record['id']] = record

22
shaker/engine/utils.py
View File

@ -57,8 +57,12 @@ def init_config_and_logging(opts):
conf.register_cli_opts(opts)
conf.register_opts(opts)
logging.register_options(conf)
logging.set_defaults(
default_log_levels=conf.default_log_levels + ['pykwalify=INFO'])
# requests to OpenStack services should be visible at DEBUG level
default_log_levels = [l for l in conf.default_log_levels
if not l.startswith('keystoneauth')]
default_log_levels += ['pykwalify=INFO']
logging.set_defaults(default_log_levels=default_log_levels)
try:
conf(project='shaker')
@ -250,3 +254,17 @@ def copy_value_by_path(src, src_param, dst, dst_param):
set_value_by_path(dst, dst_param, v)
return True
return False
def pack_openstack_params(conf):
params = dict(auth=dict(username=cfg.CONF.os_username,
password=cfg.CONF.os_password,
auth_url=cfg.CONF.os_auth_url),
os_region_name=cfg.CONF.os_region_name,
os_cacert=cfg.CONF.os_cacert,
os_insecure=cfg.CONF.os_insecure)
if cfg.CONF.os_tenant_name:
params['auth']['tenant_name'] = cfg.CONF.os_tenant_name
if cfg.CONF.os_project_name:
params['auth']['project_name'] = cfg.CONF.os_project_name
return params

15
shaker/openstack/clients/glance.py
View File

@ -13,21 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from glanceclient import client as glance_client_pkg
GLANCE_VERSION = '1'
def create_client(keystone_client, os_region_name, cacert, insecure):
image_api_url = keystone_client.service_catalog.url_for(
service_type='image', region_name=os_region_name)
return glance_client_pkg.Client(GLANCE_VERSION,
endpoint=image_api_url,
token=keystone_client.auth_token,
cacert=cacert,
insecure=insecure)
def get_image(glance_client, image_name):
for image in glance_client.images.list():

14
shaker/openstack/clients/heat.py
View File

@ -15,26 +15,12 @@
import time
from heatclient import client as heat_client_pkg
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
HEAT_VERSION = '1'
def create_client(keystone_client, os_region_name, cacert, insecure):
orchestration_api_url = keystone_client.service_catalog.url_for(
service_type='orchestration', region_name=os_region_name)
return heat_client_pkg.Client(HEAT_VERSION,
endpoint=orchestration_api_url,
token=keystone_client.auth_token,
ca_file=cacert,
insecure=insecure)
def create_stack(heat_client, stack_name, template, parameters):
stack_params = {
'stack_name': stack_name,

37
shaker/openstack/clients/keystone.py
View File

@ -1,37 +0,0 @@
# Copyright (c) 2015 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from keystoneclient.auth.identity import v2 as auth_v2
from keystoneclient import discover as keystone_discover
from keystoneclient import session
from keystoneclient.v2_0 import client as keystone_v2
from keystoneclient.v3 import client as keystone_v3
def create_keystone_client(**kwargs):
discover = keystone_discover.Discover(**kwargs)
for version_data in discover.version_data():
version = version_data["version"]
if version[0] <= 2:
return keystone_v2.Client(**kwargs)
elif version[0] == 3:
return keystone_v3.Client(**kwargs)
raise Exception(
'Failed to discover keystone version for url %(auth_url)s.', **kwargs)
def create_keystone_session(cacert, insecure, **kwargs):
auth = auth_v2.Password(**kwargs)
return session.Session(auth=auth, cert=cacert, verify=not insecure)

10
shaker/openstack/clients/neutron.py
View File

@ -13,22 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from neutronclient.neutron import client as neutron_client_pkg
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
NEUTRON_VERSION = '2.0'
def create_client(keystone_session, os_region_name):
return neutron_client_pkg.Client(NEUTRON_VERSION,
session=keystone_session,
region_name=os_region_name)
def choose_external_net(neutron_client):
ext_nets = neutron_client.list_networks(
**{'router:external': True})['networks']

7
shaker/openstack/clients/nova.py
View File

@ -23,18 +23,11 @@ from oslo_log import log as logging
LOG = logging.getLogger(__name__)
NOVA_VERSION = '2'
class ForbiddenException(nova_client_pkg.exceptions.Forbidden):
pass
def create_client(keystone_session, os_region_name):
return nova_client_pkg.Client(NOVA_VERSION, session=keystone_session,
region_name=os_region_name)
def get_available_compute_nodes(nova_client):
try:
return [dict(host=svc.host, zone=svc.zone)

95
shaker/openstack/clients/openstack.py
View File

@ -13,90 +13,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
import time
from shaker.openstack.clients import glance
from shaker.openstack.clients import heat
from shaker.openstack.clients import keystone
from shaker.openstack.clients import neutron
from shaker.openstack.clients import nova
import os_client_config
from oslo_log import log as logging
# As of now only Nova and Neutron clients support Keystone sessions.
# Thus the only way to create clients is from keystone client instance
# and auth token. The token gets expired in an hour and there is no
# way to automatically refresh it. So the current implementation is to
# recreate keystone client from scratch
MODERN_CLIENT_MAKERS = {
'neutron': neutron.create_client,
'nova': nova.create_client,
}
OLD_CLIENT_MAKERS = {
'glance': glance.create_client,
'heat': heat.create_client,
}
KEYSTONE_AUTH_EXPIRATION = 60
class OpenStackClientProxy(object):
def __init__(self, keystone_creator, client_creator):
self.keystone_creator = keystone_creator
self.client_creator = client_creator
self.last_update_time = 0
def __getattribute__(self, name):
if name in ['keystone_creator', 'client_creator',
'client', 'last_update_time']:
return super(OpenStackClientProxy, self).__getattribute__(name)
else:
now = int(time.time())
if now > self.last_update_time + KEYSTONE_AUTH_EXPIRATION:
self.last_update_time = now
self.client = self.client_creator(
keystone_client=self.keystone_creator())
return self.client.__getattribute__(name)
LOG = logging.getLogger(__name__)
class OpenStackClient(object):
def __init__(self, username, password, tenant_name, auth_url, region_name,
cacert, insecure):
self.region_name = region_name or 'RegionOne'
self.cacert = cacert or ''
self.insecure = insecure or False
self._osc_cache = {}
self.keystone_creator = functools.partial(
keystone.create_keystone_client,
username=username, password=password,
tenant_name=tenant_name, auth_url=auth_url, cacert=cacert,
insecure=insecure)
self.session_creator = functools.partial(
keystone.create_keystone_session, cacert,
username=username, password=password,
tenant_name=tenant_name, auth_url=auth_url,
insecure=insecure)
# ping OpenStack
self.keystone_creator()
def __init__(self, openstack_params):
LOG.debug('Establishing connection to OpenStack')
def __getattribute__(self, name):
if name != '_osc_cache' and name in self._osc_cache:
return self._osc_cache[name]
config = os_client_config.OpenStackConfig()
cloud_config = config.get_one_cloud(**openstack_params)
client = None
if name in MODERN_CLIENT_MAKERS:
session = self.session_creator()
client = MODERN_CLIENT_MAKERS[name](session, self.region_name)
elif name in OLD_CLIENT_MAKERS:
client_creator = functools.partial(
OLD_CLIENT_MAKERS[name], os_region_name=self.region_name,
cacert=self.cacert, insecure=self.insecure)
client = OpenStackClientProxy(self.keystone_creator,
client_creator)
self.keystone_session = cloud_config.get_session()
self.nova = cloud_config.get_legacy_client('compute')
self.neutron = cloud_config.get_legacy_client('network')
self.glance = cloud_config.get_legacy_client('image')
self.heat = cloud_config.get_legacy_client('orchestration')
if client:
self._osc_cache[name] = client
return client
else:
return super(OpenStackClient, self).__getattribute__(name)
# Ping OpenStack
self.keystone_session.get_token()
LOG.info('Connection to OpenStack is initialized')

9
shaker/tests/test_server.py
View File

@ -184,10 +184,13 @@ class TestServerPlayScenario(testtools.TestCase):
deploy_obj.deploy.assert_called_once_with(
self.deployment, base_dir='folder',
server_endpoint='127.0.0.1:5999')
openstack_params = dict(
auth=dict(username='user', password='password',
tenant_name='tenant', auth_url='auth-url'),
os_region_name='RegionOne',
os_cacert=None, os_insecure=False)
deploy_obj.connect_to_openstack.assert_called_once_with(
'user', 'password', 'tenant', 'auth-url', 'RegionOne', None,
'shaker-flavor', 'shaker-image', None, False
)
openstack_params, 'shaker-flavor', 'shaker-image', None)
deploy_obj.cleanup.assert_called_once_with()
@mock.patch('shaker.engine.deploy.Deployment')

12
test-requirements.txt
View File

@ -3,12 +3,12 @@
# process, which may cause wedges in the gate later.
# Hacking already pins down pep8, pyflakes and flake8
coverage>=3.6
coverage>=3.6 # Apache-2.0
hacking<0.11,>=0.10
mock>=1.2
mock>=1.2 # BSD
oslotest>=1.10.0 # Apache-2.0
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
sphinxcontrib-httpdomain
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
sphinxcontrib-httpdomain # BSD
sphinx_rtd_theme
testrepository>=0.0.18
testtools>=1.4.0
testrepository>=0.0.18 # Apache-2.0/BSD
testtools>=1.4.0 # MIT