Add support of resource logging, enhance cleanup script

Change-Id: I63fffaa652ab7fb0984cc057a9fc8897feb04511

.gitignore

@@ -55,6 +55,7 @@ ChangeLog
 .ropeproject/
 
 # KloudBuster
+kb*.log
 *.json
 *.html
 *.qcow2

kloudbuster/base_compute.py

@@ -19,7 +19,6 @@ import log as logging
 
 LOG = logging.getLogger(__name__)
 
-
 class BaseCompute(object):
     """
     The Base class for nova compute resources
@@ -30,6 +29,7 @@ class BaseCompute(object):
     def __init__(self, vm_name, network):
         self.novaclient = network.router.user.nova_client
         self.network = network
+        self.res_logger = network.res_logger
         self.vm_name = vm_name
         self.instance = None
         self.host = None
@@ -71,6 +71,7 @@ class BaseCompute(object):
                                                   userdata=user_data,
                                                   config_drive=config_drive,
                                                   security_groups=[sec_group.id])
+        self.res_logger.log('instances', self.vm_name, instance.id)
 
         if not instance:
             return None
@@ -79,8 +80,8 @@ class BaseCompute(object):
             instance = self.novaclient.servers.get(instance.id)
             if instance.status == 'ACTIVE':
                 self.instance = instance
-                if 'OS-EXT-SRV-ATTR:hypervisor_hostname' in instance.__dict__:
+                if 'OS-EXT-SRV-ATTR:hypervisor_hostname' in vars(instance):
-                    self.host = instance.__dict__['OS-EXT-SRV-ATTR:hypervisor_hostname']
+                    self.host = vars(instance)['OS-EXT-SRV-ATTR:hypervisor_hostname']
                 else:
                     self.host = "Unknown"
                 return instance
@@ -217,6 +218,15 @@ class Flavor(object):
     def __init__(self, novaclient):
         self.novaclient = novaclient
 
+    def get(self, name):
+        flavor = None
+        try:
+            flavor = vars(self.novaclient.flavors.find(name=name))
+        except Exception:
+            pass
+
+        return flavor
+
     def list(self):
         return self.novaclient.flavors.list()
 
@@ -240,7 +250,7 @@ class NovaQuota(object):
         self.tenant_id = tenant_id
 
     def get(self):
-        return self.novaclient.quotas.get(self.tenant_id).__dict__
+        return vars(self.novaclient.quotas.get(self.tenant_id))
 
     def update_quota(self, **kwargs):
         self.novaclient.quotas.update(self.tenant_id, **kwargs)
@@ -252,7 +262,7 @@ class CinderQuota(object):
         self.tenant_id = tenant_id
 
     def get(self):
-        return self.cinderclient.quotas.get(self.tenant_id).__dict__
+        return vars(self.cinderclient.quotas.get(self.tenant_id))
 
     def update_quota(self, **kwargs):
         self.cinderclient.quotas.update(self.tenant_id, **kwargs)

kloudbuster/base_network.py

@@ -89,6 +89,7 @@ class BaseNetwork(object):
         self.neutron_client = router.user.neutron_client
         self.nova_client = router.user.nova_client
         self.router = router
+        self.res_logger = router.res_logger
         self.network = None
         self.instance_list = []
         self.secgroup_list = []
@@ -106,6 +107,8 @@ class BaseNetwork(object):
             self.secgroup_list.append(secgroup_instance)
             secgroup_name = network_prefix + "-SG" + str(secgroup_count)
             secgroup_instance.create_secgroup_with_rules(secgroup_name)
+            self.res_logger.log('sec_groups', secgroup_instance.secgroup.name,
+                                secgroup_instance.secgroup.id)
 
         LOG.info("Scheduled to create VM for network %s..." % network_prefix)
         if config_scale['use_floatingip']:
@@ -122,6 +125,9 @@ class BaseNetwork(object):
                 # store it and the ip address in perf_instance object
                 perf_instance.fip = create_floating_ip(self.neutron_client, external_network)
                 perf_instance.fip_ip = perf_instance.fip['floatingip']['floating_ip_address']
+                self.res_logger.log('floating_ips',
+                                    perf_instance.fip['floatingip']['floating_ip_address'],
+                                    perf_instance.fip['floatingip']['id'])
 
             # Create the VMs on specified network, first keypair, first secgroup
             perf_instance.boot_info['image_name'] = config_scale['image_name']
@@ -218,6 +224,7 @@ class Router(object):
         self.nova_client = user.nova_client
         self.router = None
         self.user = user
+        self.res_logger = user.res_logger
         # Stores the list of networks
         self.network_list = []
         # Store the shared network
@@ -238,6 +245,8 @@ class Router(object):
             # Create the network and subnet
             network_name = self.router['router']['name'] + "-N" + str(network_count)
             network_instance.create_network_and_subnet(network_name)
+            self.res_logger.log('networks', network_instance.network['name'],
+                                network_instance.network['id'])
             # Attach the created network to router interface
             self.attach_router_interface(network_instance)
             # Create the compute resources in the network

kloudbuster/force_cleanup

@@ -1,82 +0,0 @@
-#! /bin/bash
-# Copyright 2015 Cisco Systems, Inc.  All rights reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-#
-
-# A tool that can represent KloudBuster json results in
-# a nicer form using HTML5, bootstrap.js and the Google Charts Javascript library
-
-###############################################################################
-#                                                                             #
-# This is a helper script which will delete all resources created by          #
-# KloudBuster.                                                                #
-#                                                                             #
-# Normally, KloudBuster will clean up automatically when it is done. However, #
-# sometimes errors or timeouts happen during the rescource creation stage,    #
-# which will cause KloudBuster out of sync with the real environment. If that #
-# happens, a force cleanup may be needed.                                     #
-#                                                                             #
-# This script will simply grep the resource name with "KB" and delete them.   #
-# If running on a production network, please double and triple check all      #
-# resources names are *NOT( containing "KB", otherwise they will be deleted   #
-# when running with this script.                                              #
-#                                                                             #
-###############################################################################
-
-# WARNING! WARNING! WARNING!
-# IMPORTANT FOR RUNNING KLOUDBUSTER ON PRODUCTION CLOUDS
-# ======================================================
-#
-# DOUBLE CHECK THE NAMES OF ALL RESOURCES THAT DOES NOT BELONG TO KLOUDBUSTER 
-# ARE *NOT* CONTAINING "KB"
-
-for line in `nova list --all-tenants | grep KB  | cut -d'|' -f2`; do
-    nova delete $line
-done
-
-echo -e "`neutron floatingip-list | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'`" | while read line; do
-    fid=`echo $line | cut -d'|' -f2 | xargs`
-    portid=`echo $line | cut -d'|' -f5 | xargs`
-    if [ "$fid" != "" ] && [ "$portid" = "" ]; then
-        neutron floatingip-delete $fid &
-    fi
-done;
-
-for line in `neutron security-group-list | grep KB | cut -d'|' -f2`; do
-    neutron security-group-delete $line &
-done;
-
-for line in `nova flavor-list | grep kb | cut -d'|' -f3`; do
-    nova flavor-delete $line &
-done;
-
-for line in `neutron router-list | grep KB | cut -d'|' -f2`; do
-    neutron router-gateway-clear $line
-    for line2 in `neutron router-port-list $line | grep subnet | cut -d'"' -f4`; do
-        neutron router-interface-delete $line $line2
-    done
-    neutron router-delete $line
-done
-
-for line in `neutron net-list | grep KB | cut -d'|' -f2`; do
-    neutron net-delete $line
-done
-
-for line in `keystone tenant-list | grep KB | cut -d'|' -f2`; do
-    keystone tenant-delete $line
-done
-
-for line in `keystone user-list | grep KB | cut -d'|' -f2`; do
-    keystone user-delete $line
-done

kloudbuster/force_cleanup.sh

@@ -0,0 +1,126 @@
+#! /bin/bash
+# Copyright 2015 Cisco Systems, Inc.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+###############################################################################
+#                                                                             #
+# This is a helper script which will delete all resources created by          #
+# KloudBuster.                                                                #
+#                                                                             #
+# Normally, KloudBuster will clean up automatically when it is done. However, #
+# sometimes errors or timeouts happen during the rescource creation stage,    #
+# which will cause KloudBuster out of sync with the real environment. If that #
+# happens, a force cleanup may be needed.                                     #
+#                                                                             #
+# It is safe to use the script with the resource list generated by            #
+# KloudBuster, usage:                                                         #
+#     $ source <rc_of_the_openstack_cloud>                                    #
+#     $ ./force_cleanup --file kb_20150807_183001_svr.log                     #
+#     $ ./force_cleanup --file kb_20150807_183001_cnt.log                     #
+#                                                                             #
+# Note: If running under single-tenant or tenant/user reusing mode, you have  #
+#       to cleanup the server resources first, then client resources.         #
+#                                                                             #
+# When there is no resource list provided, the script will simply grep the    #
+# resource name with "KB" and delete them. If running on a production         #
+# network, please double and triple check all resources names are *NOT*       #
+# containing "KB", otherwise they will be deleted by the script.              #
+#                                                                             #
+###############################################################################
+
+# ======================================================
+#                        WARNING
+# ======================================================
+# IMPORTANT FOR RUNNING KLOUDBUSTER ON PRODUCTION CLOUDS
+#
+# DOUBLE CHECK THE NAMES OF ALL RESOURCES THAT DO NOT
+# BELONG TO KLOUDBUSTER ARE *NOT* CONTAINING "KB".
+# ======================================================
+function prompt_to_run() {
+    echo "Warning: You didn't specify a resource list file as the input,"\
+         "or the input file is invalid. The script will delete all"\
+         "resources of the cloud whose names contain \"KB\". "
+    read -p "Are you sure? (Y/N) " answer
+    if [ "$answer" != "Y" ] && [ "$answer" != "y" ]; then
+        exit 0
+    fi
+}
+
+if [ "$1" == "--file" ] && [ -f "$2" ]; then
+    INSTANCE_LIST=`grep "instances" $2 | cut -d'|' -f3`
+    SEC_GROUP_LIST=`grep "sec_groups" $2 | cut -d'|' -f3`
+    FLAVOR_LIST=`grep "flavors" $2 | cut -d'|' -f3`
+    ROUTER_LIST=`grep "routers" $2 | cut -d'|' -f3`
+    NETWORK_LIST=`grep "networks" $2 | cut -d'|' -f3`
+    TENANT_LIST=`grep "tenants" $2 | cut -d'|' -f3`
+    USER_LIST=`grep "users" $2 | cut -d'|' -f3`
+    FLOATINGIP_LIST=`grep "floating_ips" $2 | cut -d'|' -f3`
+else
+    prompt_to_run;
+    INSTANCE_LIST=`nova list --all-tenants | grep KB  | cut -d'|' -f2`
+    SEC_GROUP_LIST=`neutron security-group-list | grep KB | cut -d'|' -f2`
+    FLAVOR_LIST=`nova flavor-list | grep kb | cut -d'|' -f3`
+    ROUTER_LIST=`neutron router-list | grep KB | cut -d'|' -f2`
+    NETWORK_LIST=`neutron net-list | grep KB | cut -d'|' -f2`
+    TENANT_LIST=`keystone tenant-list | grep KB | cut -d'|' -f2`
+    USER_LIST=`keystone user-list | grep KB | cut -d'|' -f2`
+    FLOATINGIP_LIST=""
+fi
+
+for line in $INSTANCE_LIST; do
+    nova delete $line
+done
+
+for line in $FLAVOR_LIST; do
+    nova flavor-delete $line
+done;
+
+for line in $SEC_GROUP_LIST; do
+    neutron security-group-delete $line &
+done;
+
+if [ "$FLOATINGIP_LIST" == "" ]; then
+    echo -e "`neutron floatingip-list | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'`" | while read line; do
+        fid=`echo $line | cut -d'|' -f2 | xargs`
+        portid=`echo $line | cut -d'|' -f5 | xargs`
+        if [ "$fid" != "" ] && [ "$portid" = "" ]; then
+            neutron floatingip-delete $fid
+        fi
+    done;
+else
+    for line in $FLOATINGIP_LIST; do
+        neutron floatingip-delete $line &
+    done;
+fi
+
+for line in $ROUTER_LIST; do
+    neutron router-gateway-clear $line
+    for line2 in `neutron router-port-list $line | grep subnet_id | cut -d'"' -f4`; do
+        neutron router-interface-delete $line $line2
+    done
+    neutron router-delete $line
+done
+
+for line in $NETWORK_LIST; do
+    neutron net-delete $line
+done
+
+for line in $TENANT_LIST; do
+    keystone tenant-delete $line
+done
+
+for line in $USER_LIST; do
+    keystone user-delete $line
+done

kloudbuster/kb_res_logger.py

@@ -0,0 +1,50 @@
+# Copyright 2015 Cisco Systems, Inc.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import log as logging
+from time import gmtime
+from time import strftime
+
+LOG = logging.getLogger(__name__)
+
+class KBResTypeInvalid(Exception):
+    pass
+
+class KBResLogger(object):
+
+    def __init__(self):
+        self.resource_list = {}
+        for key in ['tenants', 'users', 'flavors', 'routers', 'networks',
+                    'sec_groups', 'instances', 'floating_ips']:
+            self.resource_list[key] = []
+
+    def log(self, res_type, name, id):
+        if res_type in self.resource_list:
+            self.resource_list[res_type].append({'name': name, 'id': id})
+        else:
+            raise KBResTypeInvalid()
+
+    @staticmethod
+    def dump_and_save(role, res_dict):
+        res_text = ""
+        filename = "kb_%s_%s.log" % (strftime("%Y%m%d_%H%M%S", gmtime()), role)
+        for key in res_dict:
+            for item in res_dict[key]:
+                line = "%s|%s|%s\n" % (key, item['name'], item['id'])
+                res_text = res_text + line
+
+        fout = open(filename, 'w')
+        fout.write(res_text)
+        fout.close()
+        LOG.info('Resources list is saved to file: %s.', filename)

kloudbuster/kloudbuster.py

@@ -26,6 +26,7 @@ import base_network
 import glanceclient.exc as glance_exception
 from glanceclient.v2 import client as glanceclient
 from kb_config import KBConfig
+from kb_res_logger import KBResLogger
 from kb_runner import KBRunner
 from kb_scheduler import KBScheduler
 from keystoneclient.v2_0 import client as keystoneclient
@@ -44,11 +45,11 @@ class KBVMCreationException(Exception):
     pass
 
 
-def create_keystone_client(admin_creds):
+def create_keystone_client(creds):
     """
     Return the keystone client and auth URL given a credential
     """
-    creds = admin_creds.get_credentials()
+    creds = creds.get_credentials()
     return (keystoneclient.Client(**creds), creds['auth_url'])
 
 def check_and_upload_images(cred, cred_testing, server_img_name, client_img_name):
@@ -67,7 +68,7 @@ def check_and_upload_images(cred, cred_testing, server_img_name, client_img_name
         except StopIteration:
             pass
 
-        # Trying upload images
+        # Trying to upload images
         LOG.info("Image is not found in %s, trying to upload..." % (kloud))
         if not os.path.exists('dib/kloudbuster.qcow2'):
             LOG.error("Image file dib/kloudbuster.qcow2 is not present, please refer "
@@ -88,14 +89,14 @@ def check_and_upload_images(cred, cred_testing, server_img_name, client_img_name
     return True
 
 class Kloud(object):
-    def __init__(self, scale_cfg, admin_creds, reusing_tenants, testing_side=False):
+    def __init__(self, scale_cfg, cred, reusing_tenants, testing_side=False):
-        self.cred = admin_creds
         self.tenant_list = []
         self.testing_side = testing_side
         self.scale_cfg = scale_cfg
         self.reusing_tenants = reusing_tenants
-        self.keystone, self.auth_url = create_keystone_client(self.cred)
+        self.keystone, self.auth_url = create_keystone_client(cred)
         self.flavor_to_use = None
+        self.res_logger = KBResLogger()
         if testing_side:
             self.prefix = 'KBc'
             self.name = 'Client Kloud'
@@ -103,9 +104,6 @@ class Kloud(object):
             self.prefix = 'KBs'
             self.name = 'Server Kloud'
         LOG.info("Creating kloud: " + self.prefix)
-        # if this cloud is sharing a network then all tenants must hook up to
-        # it and on deletion that shared network must NOT be deleted
-        # as it will be deleted by the owner
 
         # pre-compute the placement az to use for all VMs
         self.placement_az = None
@@ -125,6 +123,8 @@ class Kloud(object):
             for tenant_count in xrange(self.scale_cfg['number_tenants']):
                 tenant_name = self.prefix + "-T" + str(tenant_count)
                 tenant_instance = tenant.Tenant(tenant_name, self, tenant_quota)
+                self.res_logger.log('tenants', tenant_instance.tenant_name,
+                                    tenant_instance.tenant_id)
                 self.tenant_list.append(tenant_instance)
 
         for tenant_instance in self.tenant_list:
@@ -136,10 +136,14 @@ class Kloud(object):
             flavor_manager = base_compute.Flavor(nova_client)
             flavor_dict = self.scale_cfg.flavor
             if self.testing_side:
-                flavor_manager.create_flavor('kb.client', override=True, **flavor_dict)
+                flv = flavor_manager.create_flavor('kb.client', override=True, **flavor_dict)
-                flavor_manager.create_flavor('kb.proxy', override=True, ram=2048, vcpus=1, disk=20)
+                self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id'])
+                flv = flavor_manager.create_flavor('kb.proxy', override=True,
+                                                   ram=2048, vcpus=1, disk=20)
+                self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id'])
             else:
-                flavor_manager.create_flavor('kb.server', override=True, **flavor_dict)
+                flv = flavor_manager.create_flavor('kb.server', override=True, **flavor_dict)
+                self.res_logger.log('flavors', vars(flv)['name'], vars(flv)['id'])
 
     def delete_resources(self):
         # Deleting flavors created by KloudBuster
@@ -206,6 +210,9 @@ class Kloud(object):
             external_network = base_network.find_external_network(neutron_client)
             instance.fip = base_network.create_floating_ip(neutron_client, external_network)
             instance.fip_ip = instance.fip['floatingip']['floating_ip_address']
+            self.res_logger.log('floating_ips',
+                                instance.fip['floatingip']['floating_ip_address'],
+                                instance.fip['floatingip']['id'])
 
         if instance.fip:
             # Associate the floating ip with this instance
@@ -318,8 +325,6 @@ class KloudBuster(object):
     def run(self):
         """
         The runner for KloudBuster Tests
-        Executes tests serially
-        Support concurrency in fututure
         """
         kbrunner = None
         vm_creation_concurrency = self.client_cfg.vm_creation_concurrency
@@ -401,11 +406,13 @@ class KloudBuster(object):
                 self.kloud.delete_resources()
             except Exception:
                 traceback.print_exc()
+                KBResLogger.dump_and_save('svr', self.kloud.res_logger.resource_list)
         if self.client_cfg['cleanup_resources']:
             try:
                 self.testing_kloud.delete_resources()
             except Exception:
                 traceback.print_exc()
+                KBResLogger.dump_and_save('clt', self.testing_kloud.res_logger.resource_list)
         if kbrunner:
             kbrunner.dispose()
 
@@ -506,32 +513,7 @@ class KloudBuster(object):
 
         return quota_dict
 
-# Some hardcoded client side options we do not want users to change
+def main():
-hardcoded_client_cfg = {
-    # Number of tenants to be created on the cloud
-    'number_tenants': 1,
-
-    # Number of Users to be created inside the tenant
-    'users_per_tenant': 1,
-
-    # Number of routers to be created within the context of each User
-    # For now support only 1 router per user
-    'routers_per_user': 1,
-
-    # Number of networks to be created within the context of each Router
-    # Assumes 1 subnet per network
-    'networks_per_router': 1,
-
-    # Number of VM instances to be created within the context of each Network
-    'vms_per_network': 1,
-
-    # Number of security groups per network
-    'secgroups_per_network': 1
-}
-
-
-if __name__ == '__main__':
-
     cli_opts = [
         cfg.StrOpt("config",
                    short="c",
@@ -596,3 +578,7 @@ if __name__ == '__main__':
         LOG.info('Saving results in json file: ' + CONF.json + "...")
         with open(CONF.json, 'w') as jfp:
             json.dump(kloudbuster.final_result, jfp, indent=4, sort_keys=True)
+
+
+if __name__ == '__main__':
+    main()

kloudbuster/tenant.py

@@ -35,6 +35,7 @@ class Tenant(object):
         tested cloud being on same cloud
         """
         self.kloud = kloud
+        self.res_logger = kloud.res_logger
         self.tenant_name = tenant_name
         if not self.kloud.reusing_tenants:
             self.tenant_object = self._get_tenant()
@@ -97,6 +98,7 @@ class Tenant(object):
                                            self.kloud.scale_cfg['keystone_admin_role'])
                 # Global list with all user instances
                 self.user_list.append(user_instance)
+                self.res_logger.log('users', user_instance.user_name, user_instance.user.id)
 
         for user_instance in self.user_list:
             # Now create the user resources like routers which inturn trigger network and

kloudbuster/users.py

@@ -45,6 +45,7 @@ class User(object):
         self.user_name = user_name
         self.password = password
         self.tenant = tenant
+        self.res_logger = tenant.res_logger
         self.router_list = []
         # Store the nova, neutron and cinder client
         self.nova_client = None
@@ -133,7 +134,7 @@ class User(object):
         flavor_manager = base_compute.Flavor(self.nova_client)
         flavor_to_use = None
         for flavor in flavor_manager.list():
-            flavor = flavor.__dict__
+            flavor = vars(flavor)
             if flavor['vcpus'] < 1 or flavor['ram'] < 1024 or flavor['disk'] < 10:
                 continue
             flavor_to_use = flavor
@@ -221,6 +222,8 @@ class User(object):
             router_name = self.user_name + "-R" + str(router_count)
             # Create the router and also attach it to external network
             router_instance.create_router(router_name, external_network)
+            self.res_logger.log('routers', router_instance.router['router']['name'],
+                                router_instance.router['router']['id'])
             # Now create the network resources inside the router
             router_instance.create_network_resources(config_scale)
 

kloudbuster/wrk_tool.py

@@ -121,7 +121,6 @@ class WrkTool(PerfTool):
                     all_res[key] += item['results'][key]
             all_res[key] = int(all_res[key])
 
-
         if 'latency_stats' in results[0]['results']:
             # for item in results:
             #     print item['results']['latency_stats']

setup.cfg

@@ -23,6 +23,10 @@ classifier =
 packages =
     kloudbuster
 
+[entry_points]
+console_scripts =
+    kloudbuster = kloudbuster.kloudbuster:main
+
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build