x/kloudbuster
Code Issues Proposed changes

Add support of resource logging, enhance cleanup script

Browse Source 
Change-Id: I63fffaa652ab7fb0984cc057a9fc8897feb04511
This commit is contained in:
Yichen Wang 2015-08-07 13:20:22 -07:00
parent f11f8d3a6c
commit 55d800bd1c
11 changed files with 237 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -55,6 +55,7 @@ ChangeLog
.ropeproject/
# KloudBuster
kb*.log
*.json
*.html
*.qcow2

20
kloudbuster/base_compute.py
View File

@ -19,7 +19,6 @@ import log as logging
LOG = logging.getLogger(__name__)
class BaseCompute(object):
"""
The Base class for nova compute resources
@ -30,6 +29,7 @@ class BaseCompute(object):
def __init__(self, vm_name, network):
self.novaclient = network.router.user.nova_client
self.network = network
self.res_logger = network.res_logger
self.vm_name = vm_name
self.instance = None
self.host = None
@ -71,6 +71,7 @@ class BaseCompute(object):
userdata=user_data,
config_drive=config_drive,
security_groups=[sec_group.id])
self.res_logger.log('instances', self.vm_name, instance.id)
if not instance:
return None
@ -79,8 +80,8 @@ class BaseCompute(object):
instance = self.novaclient.servers.get(instance.id)
if instance.status == 'ACTIVE':
self.instance = instance
if 'OS-EXT-SRV-ATTR:hypervisor_hostname' in instance.__dict__:
self.host = instance.__dict__['OS-EXT-SRV-ATTR:hypervisor_hostname']
if 'OS-EXT-SRV-ATTR:hypervisor_hostname' in vars(instance):
self.host = vars(instance)['OS-EXT-SRV-ATTR:hypervisor_hostname']
else:
self.host = "Unknown"
return instance
@ -217,6 +218,15 @@ class Flavor(object):
def __init__(self, novaclient):
self.novaclient = novaclient
def get(self, name):
flavor = None
try:
flavor = vars(self.novaclient.flavors.find(name=name))
except Exception:
pass
return flavor
def list(self):
return self.novaclient.flavors.list()
@ -240,7 +250,7 @@ class NovaQuota(object):
self.tenant_id = tenant_id
def get(self):
return self.novaclient.quotas.get(self.tenant_id).__dict__
return vars(self.novaclient.quotas.get(self.tenant_id))
def update_quota(self, **kwargs):
self.novaclient.quotas.update(self.tenant_id, **kwargs)
@ -252,7 +262,7 @@ class CinderQuota(object):
self.tenant_id = tenant_id
def get(self):
return self.cinderclient.quotas.get(self.tenant_id).__dict__
return vars(self.cinderclient.quotas.get(self.tenant_id))
def update_quota(self, **kwargs):
self.cinderclient.quotas.update(self.tenant_id, **kwargs)

9
kloudbuster/base_network.py
View File

@ -89,6 +89,7 @@ class BaseNetwork(object):
self.neutron_client = router.user.neutron_client
self.nova_client = router.user.nova_client
self.router = router
self.res_logger = router.res_logger
self.network = None
self.instance_list = []
self.secgroup_list = []
@ -106,6 +107,8 @@ class BaseNetwork(object):
self.secgroup_list.append(secgroup_instance)
secgroup_name = network_prefix + "-SG" + str(secgroup_count)
secgroup_instance.create_secgroup_with_rules(secgroup_name)
self.res_logger.log('sec_groups', secgroup_instance.secgroup.name,
secgroup_instance.secgroup.id)
LOG.info("Scheduled to create VM for network %s..." % network_prefix)
if config_scale['use_floatingip']:
@ -122,6 +125,9 @@ class BaseNetwork(object):
# store it and the ip address in perf_instance object
perf_instance.fip = create_floating_ip(self.neutron_client, external_network)
perf_instance.fip_ip = perf_instance.fip['floatingip']['floating_ip_address']
self.res_logger.log('floating_ips',
perf_instance.fip['floatingip']['floating_ip_address'],
perf_instance.fip['floatingip']['id'])
# Create the VMs on specified network, first keypair, first secgroup
perf_instance.boot_info['image_name'] = config_scale['image_name']
@ -218,6 +224,7 @@ class Router(object):
self.nova_client = user.nova_client
self.router = None
self.user = user
self.res_logger = user.res_logger
# Stores the list of networks
self.network_list = []
# Store the shared network
@ -238,6 +245,8 @@ class Router(object):
# Create the network and subnet
network_name = self.router['router']['name'] + "-N" + str(network_count)
network_instance.create_network_and_subnet(network_name)
self.res_logger.log('networks', network_instance.network['name'],
network_instance.network['id'])
# Attach the created network to router interface
self.attach_router_interface(network_instance)
# Create the compute resources in the network

82
kloudbuster/force_cleanup
View File

@ -1,82 +0,0 @@
#! /bin/bash
# Copyright 2015 Cisco Systems, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# A tool that can represent KloudBuster json results in
# a nicer form using HTML5, bootstrap.js and the Google Charts Javascript library
###############################################################################
# #
# This is a helper script which will delete all resources created by #
# KloudBuster. #
# #
# Normally, KloudBuster will clean up automatically when it is done. However, #
# sometimes errors or timeouts happen during the rescource creation stage, #
# which will cause KloudBuster out of sync with the real environment. If that #
# happens, a force cleanup may be needed. #
# #
# This script will simply grep the resource name with "KB" and delete them. #
# If running on a production network, please double and triple check all #
# resources names are *NOT( containing "KB", otherwise they will be deleted #
# when running with this script. #
# #
###############################################################################
# WARNING! WARNING! WARNING!
# IMPORTANT FOR RUNNING KLOUDBUSTER ON PRODUCTION CLOUDS
# ======================================================
#
# DOUBLE CHECK THE NAMES OF ALL RESOURCES THAT DOES NOT BELONG TO KLOUDBUSTER
# ARE *NOT* CONTAINING "KB"
for line in `nova list --all-tenants | grep KB | cut -d'|' -f2`; do
nova delete $line
done
echo -e "`neutron floatingip-list | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'`" | while read line; do
fid=`echo $line | cut -d'|' -f2 | xargs`
portid=`echo $line | cut -d'|' -f5 | xargs`
if [ "$fid" != "" ] && [ "$portid" = "" ]; then
neutron floatingip-delete $fid &
fi
done;
for line in `neutron security-group-list | grep KB | cut -d'|' -f2`; do
neutron security-group-delete $line &
done;
for line in `nova flavor-list | grep kb | cut -d'|' -f3`; do
nova flavor-delete $line &
done;
for line in `neutron router-list | grep KB | cut -d'|' -f2`; do
neutron router-gateway-clear $line
for line2 in `neutron router-port-list $line | grep subnet | cut -d'"' -f4`; do
neutron router-interface-delete $line $line2
done
neutron router-delete $line
done
for line in `neutron net-list | grep KB | cut -d'|' -f2`; do
neutron net-delete $line
done
for line in `keystone tenant-list | grep KB | cut -d'|' -f2`; do
keystone tenant-delete $line
done
for line in `keystone user-list | grep KB | cut -d'|' -f2`; do
keystone user-delete $line
done

126
kloudbuster/force_cleanup.sh Executable file
View File

@ -0,0 +1,126 @@
#! /bin/bash
# Copyright 2015 Cisco Systems, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
###############################################################################
# #
# This is a helper script which will delete all resources created by #
# KloudBuster. #
# #
# Normally, KloudBuster will clean up automatically when it is done. However, #
# sometimes errors or timeouts happen during the rescource creation stage, #
# which will cause KloudBuster out of sync with the real environment. If that #
# happens, a force cleanup may be needed. #
# #
# It is safe to use the script with the resource list generated by #
# KloudBuster, usage: #
# $ source <rc_of_the_openstack_cloud> #
# $ ./force_cleanup --file kb_20150807_183001_svr.log #
# $ ./force_cleanup --file kb_20150807_183001_cnt.log #
# #
# Note: If running under single-tenant or tenant/user reusing mode, you have #
# to cleanup the server resources first, then client resources. #
# #
# When there is no resource list provided, the script will simply grep the #
# resource name with "KB" and delete them. If running on a production #
# network, please double and triple check all resources names are *NOT* #
# containing "KB", otherwise they will be deleted by the script. #
# #
###############################################################################
# ======================================================
# WARNING
# ======================================================
# IMPORTANT FOR RUNNING KLOUDBUSTER ON PRODUCTION CLOUDS
#
# DOUBLE CHECK THE NAMES OF ALL RESOURCES THAT DO NOT
# BELONG TO KLOUDBUSTER ARE *NOT* CONTAINING "KB".
# ======================================================
function prompt_to_run() {
echo "Warning: You didn't specify a resource list file as the input,"\
"or the input file is invalid. The script will delete all"\
"resources of the cloud whose names contain \"KB\". "
read -p "Are you sure? (Y/N) " answer
if [ "$answer" != "Y" ] && [ "$answer" != "y" ]; then
exit 0
fi
}
if [ "$1" == "--file" ] && [ -f "$2" ]; then
INSTANCE_LIST=`grep "instances" $2 | cut -d'|' -f3`
SEC_GROUP_LIST=`grep "sec_groups" $2 | cut -d'|' -f3`
FLAVOR_LIST=`grep "flavors" $2 | cut -d'|' -f3`
ROUTER_LIST=`grep "routers" $2 | cut -d'|' -f3`
NETWORK_LIST=`grep "networks" $2 | cut -d'|' -f3`
TENANT_LIST=`grep "tenants" $2 | cut -d'|' -f3`
USER_LIST=`grep "users" $2 | cut -d'|' -f3`
FLOATINGIP_LIST=`grep "floating_ips" $2 | cut -d'|' -f3`
else
prompt_to_run;
INSTANCE_LIST=`nova list --all-tenants | grep KB | cut -d'|' -f2`
SEC_GROUP_LIST=`neutron security-group-list | grep KB | cut -d'|' -f2`
FLAVOR_LIST=`nova flavor-list | grep kb | cut -d'|' -f3`
ROUTER_LIST=`neutron router-list | grep KB | cut -d'|' -f2`
NETWORK_LIST=`neutron net-list | grep KB | cut -d'|' -f2`
TENANT_LIST=`keystone tenant-list | grep KB | cut -d'|' -f2`
USER_LIST=`keystone user-list | grep KB | cut -d'|' -f2`
FLOATINGIP_LIST=""
fi
for line in $INSTANCE_LIST; do
nova delete $line
done
for line in $FLAVOR_LIST; do
nova flavor-delete $line
done;
for line in $SEC_GROUP_LIST; do
neutron security-group-delete $line &
done;
if [ "$FLOATINGIP_LIST" == "" ]; then
echo -e "`neutron floatingip-list | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'`" | while read line; do
fid=`echo $line | cut -d'|' -f2 | xargs`
portid=`echo $line | cut -d'|' -f5 | xargs`
if [ "$fid" != "" ] && [ "$portid" = "" ]; then
neutron floatingip-delete $fid
fi
done;
else
for line in $FLOATINGIP_LIST; do
neutron floatingip-delete $line &
done;
fi
for line in $ROUTER_LIST; do
neutron router-gateway-clear $line
for line2 in `neutron router-port-list $line | grep subnet_id | cut -d'"' -f4`; do
neutron router-interface-delete $line $line2
done
neutron router-delete $line
done
for line in $NETWORK_LIST; do
neutron net-delete $line
done
for line in $TENANT_LIST; do
keystone tenant-delete $line
done
for line in $USER_LIST; do
keystone user-delete $line
done

50
kloudbuster/kb_res_logger.py Normal file
View File

@ -0,0 +1,50 @@
# Copyright 2015 Cisco Systems, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import log as logging
from time import gmtime
from time import strftime
LOG = logging.getLogger(__name__)
class KBResTypeInvalid(Exception):
pass
class KBResLogger(object):
def __init__(self):
self.resource_list = {}
for key in ['tenants', 'users', 'flavors', 'routers', 'networks',
'sec_groups', 'instances', 'floating_ips']:
self.resource_list[key] = []
def log(self, res_type, name, id):
if res_type in self.resource_list:
self.resource_list[res_type].append({'name': name, 'id': id})
else:
raise KBResTypeInvalid()
@staticmethod
def dump_and_save(role, res_dict):
res_text = ""
filename = "kb_%s_%s.log" % (strftime("%Y%m%d_%H%M%S", gmtime()), role)
for key in res_dict:
for item in res_dict[key]:
line = "%s|%s|%s\n" % (key, item['name'], item['id'])
res_text = res_text + line
fout = open(filename, 'w')
fout.write(res_text)
fout.close()
LOG.info('Resources list is saved to file: %s.', filename)

66
kloudbuster/kloudbuster.py
View File

@ -26,6 +26,7 @@ import base_network
import glanceclient.exc as glance_exception
from glanceclient.v2 import client as glanceclient
from kb_config import KBConfig
from kb_res_logger import KBResLogger
from kb_runner import KBRunner
from kb_scheduler import KBScheduler
from keystoneclient.v2_0 import client as keystoneclient
@ -44,11 +45,11 @@ class KBVMCreationException(Exception):
pass
def create_keystone_client(admin_creds):
def create_keystone_client(creds):
"""
Return the keystone client and auth URL given a credential
"""
creds = admin_creds.get_credentials()
creds = creds.get_credentials()
return (keystoneclient.Client(**creds), creds['auth_url'])
def check_and_upload_images(cred, cred_testing, server_img_name, client_img_name):
@ -67,7 +68,7 @@ def check_and_upload_images(cred, cred_testing, server_img_name, client_img_name
except StopIteration:
pass
# Trying upload images
# Trying to upload images
LOG.info("Image is not found in %s, trying to upload..." % (kloud))
if not os.path.exists('dib/kloudbuster.qcow2'):
LOG.error("Image file dib/kloudbuster.qcow2 is not present, please refer "
@ -88,14 +89,14 @@ def check_and_upload_images(cred, cred_testing, server_img_name, client_img_name
return True
class Kloud(object):
def __init__(self, scale_cfg, admin_creds, reusing_tenants, testing_side=False):
self.cred = admin_creds
def __init__(self, scale_cfg, cred, reusing_tenants, testing_side=False):
self.tenant_list = []
self.testing_side = testing_side
self.scale_cfg = scale_cfg
self.reusing_tenants = reusing_tenants
self.keystone, self.auth_url = create_keystone_client(self.cred)
self.keystone, self.auth_url = create_keystone_client(cred)
self.flavor_to_use = None
self.res_logger = KBResLogger()
if testing_side:
self.prefix = 'KBc'
self.name = 'Client Kloud'
@ -103,9 +104,6 @@ class Kloud(object):
self.prefix = 'KBs'
self.name = 'Server Kloud'
LOG.info("Creating kloud: " + self.prefix)
# if this cloud is sharing a network then all tenants must hook up to
# it and on deletion that shared network must NOT be deleted
# as it will be deleted by the owner
# pre-compute the placement az to use for all VMs
self.placement_az = None
@ -125,6 +123,8 @@ class Kloud(object):
for tenant_count in xrange(self.scale_cfg['number_tenants']):
tenant_name = self.prefix + "-T" + str(tenant_count)
tenant_instance = tenant.Tenant(tenant_name, self, tenant_quota)
self.res_logger.log('tenants', tenant_instance.tenant_name,
tenant_instance.tenant_id)
self.tenant_list.append(tenant_instance)
for tenant_instance in self.tenant_list:
@ -136,10 +136,14 @@ class Kloud(object):
flavor_manager = base_compute.Flavor(nova_client)
flavor_dict = self.scale_cfg.flavor
if self.testing_side:
flavor_manager.create_flavor('kb.client', override=True, **flavor_dict)
flavor_manager.create_flavor('kb.proxy', override=True, ram=2048, vcpus=1, disk=20)
flv = flavor_manager.create_flavor('kb.client', override=True, **flavor_dict)
self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id'])
flv = flavor_manager.create_flavor('kb.proxy', override=True,
ram=2048, vcpus=1, disk=20)
self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id'])
else:
flavor_manager.create_flavor('kb.server', override=True, **flavor_dict)
flv = flavor_manager.create_flavor('kb.server', override=True, **flavor_dict)
self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id'])
def delete_resources(self):
# Deleting flavors created by KloudBuster
@ -206,6 +210,9 @@ class Kloud(object):
external_network = base_network.find_external_network(neutron_client)
instance.fip = base_network.create_floating_ip(neutron_client, external_network)
instance.fip_ip = instance.fip['floatingip']['floating_ip_address']
self.res_logger.log('floating_ips',
instance.fip['floatingip']['floating_ip_address'],
instance.fip['floatingip']['id'])
if instance.fip:
# Associate the floating ip with this instance
@ -318,8 +325,6 @@ class KloudBuster(object):
def run(self):
"""
The runner for KloudBuster Tests
Executes tests serially
Support concurrency in fututure
"""
kbrunner = None
vm_creation_concurrency = self.client_cfg.vm_creation_concurrency
@ -401,11 +406,13 @@ class KloudBuster(object):
self.kloud.delete_resources()
except Exception:
traceback.print_exc()
KBResLogger.dump_and_save('svr', self.kloud.res_logger.resource_list)
if self.client_cfg['cleanup_resources']:
try:
self.testing_kloud.delete_resources()
except Exception:
traceback.print_exc()
KBResLogger.dump_and_save('clt', self.testing_kloud.res_logger.resource_list)
if kbrunner:
kbrunner.dispose()
@ -506,32 +513,7 @@ class KloudBuster(object):
return quota_dict
# Some hardcoded client side options we do not want users to change
hardcoded_client_cfg = {
# Number of tenants to be created on the cloud
'number_tenants': 1,
# Number of Users to be created inside the tenant
'users_per_tenant': 1,
# Number of routers to be created within the context of each User
# For now support only 1 router per user
'routers_per_user': 1,
# Number of networks to be created within the context of each Router
# Assumes 1 subnet per network
'networks_per_router': 1,
# Number of VM instances to be created within the context of each Network
'vms_per_network': 1,
# Number of security groups per network
'secgroups_per_network': 1
}
if __name__ == '__main__':
def main():
cli_opts = [
cfg.StrOpt("config",
short="c",
@ -596,3 +578,7 @@ if __name__ == '__main__':
LOG.info('Saving results in json file: ' + CONF.json + "...")
with open(CONF.json, 'w') as jfp:
json.dump(kloudbuster.final_result, jfp, indent=4, sort_keys=True)
if __name__ == '__main__':
main()

2
kloudbuster/tenant.py
View File

@ -35,6 +35,7 @@ class Tenant(object):
tested cloud being on same cloud
"""
self.kloud = kloud
self.res_logger = kloud.res_logger
self.tenant_name = tenant_name
if not self.kloud.reusing_tenants:
self.tenant_object = self._get_tenant()
@ -97,6 +98,7 @@ class Tenant(object):
self.kloud.scale_cfg['keystone_admin_role'])
# Global list with all user instances
self.user_list.append(user_instance)
self.res_logger.log('users', user_instance.user_name, user_instance.user.id)
for user_instance in self.user_list:
# Now create the user resources like routers which inturn trigger network and

5
kloudbuster/users.py
View File

@ -45,6 +45,7 @@ class User(object):
self.user_name = user_name
self.password = password
self.tenant = tenant
self.res_logger = tenant.res_logger
self.router_list = []
# Store the nova, neutron and cinder client
self.nova_client = None
@ -133,7 +134,7 @@ class User(object):
flavor_manager = base_compute.Flavor(self.nova_client)
flavor_to_use = None
for flavor in flavor_manager.list():
flavor = flavor.__dict__
flavor = vars(flavor)
if flavor['vcpus'] < 1 or flavor['ram'] < 1024 or flavor['disk'] < 10:
continue
flavor_to_use = flavor
@ -221,6 +222,8 @@ class User(object):
router_name = self.user_name + "-R" + str(router_count)
# Create the router and also attach it to external network
router_instance.create_router(router_name, external_network)
self.res_logger.log('routers', router_instance.router['router']['name'],
router_instance.router['router']['id'])
# Now create the network resources inside the router
router_instance.create_network_resources(config_scale)

1
kloudbuster/wrk_tool.py
View File

@ -121,7 +121,6 @@ class WrkTool(PerfTool):
all_res[key] += item['results'][key]
all_res[key] = int(all_res[key])
if 'latency_stats' in results[0]['results']:
# for item in results:
# print item['results']['latency_stats']

4
setup.cfg
View File

@ -23,6 +23,10 @@ classifier =
packages =
kloudbuster
[entry_points]
console_scripts =
kloudbuster = kloudbuster.kloudbuster:main
[build_sphinx]
source-dir = doc/source
build-dir = doc/build