Add TSDB plugin including Prometheus
Change-Id: Id832932b6b84f6d296bfd01d8fe91ad0edb0f7d3
This commit is contained in:
Kerim Gokarslan
parent
753eb84b2b
commit
51962aa0f1
@ -64,6 +64,26 @@ public_key_file:
|
||||
# Name of Provider network used for multicast tests.
|
||||
multicast_provider_network_name: '' # Leave blank to use first available (default) network.
|
||||
|
||||
# TSDB connectors are optional and can be used to retrieve CPU usage information and attach them
|
||||
# to the results.
|
||||
tsdb:
|
||||
# The default TSDB class will return nothing (i.e. feature disabled by default).
|
||||
# Must be replaced with a functional class module and name to retrieve CPU usage information from
|
||||
# the actual TSDB.
|
||||
module: 'kloudbuster.tsdb'
|
||||
# TSDB class name. This class has to be defined in the module given in tsdb_module.
|
||||
class: 'TSDB'
|
||||
# The interval in seconds between 2 consecutive CPU measurements to be returned by the TSDB.
|
||||
step_size: 30
|
||||
# Duration in seconds of the warmup and cooldown period
|
||||
# If run duration is 110 seconds, CPU metric will be retrieved for the window starting 10 sec from
|
||||
# the start (skip the warmup period) and 10 sec before the end of the run
|
||||
# (skip the cooldown period), and there will be 90/30 = 3 samples retrieved per CPU core.
|
||||
wait_time: 10
|
||||
# TSDB server address
|
||||
server_ip: localhost
|
||||
# TSDB server port
|
||||
server_port: 9090
|
||||
|
||||
# ==================================================
|
||||
# SERVER SIDE CONFIG OPTIONS (HTTP SERVER SIDE ONLY)
|
||||
|
||||
@ -27,9 +27,11 @@ import credentials
|
||||
CONF = cfg.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KBConfigParseException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# Some hardcoded client side options we do not want users to change
|
||||
hardcoded_client_cfg = {
|
||||
# Number of tenants to be created on the cloud
|
||||
@ -49,6 +51,7 @@ hardcoded_client_cfg = {
|
||||
'secgroups_per_network': 1
|
||||
}
|
||||
|
||||
|
||||
def get_absolute_path_for_file(file_name):
|
||||
'''
|
||||
Return the filename in absolute path for any file
|
||||
@ -64,8 +67,8 @@ def get_absolute_path_for_file(file_name):
|
||||
|
||||
return abs_file_path
|
||||
|
||||
class KBConfig(object):
|
||||
|
||||
class KBConfig(object):
|
||||
def __init__(self):
|
||||
# The default configuration file for KloudBuster
|
||||
default_cfg = resource_string(__name__, "cfg.scale.yaml")
|
||||
@ -80,6 +83,9 @@ class KBConfig(object):
|
||||
self.tenants_list = None
|
||||
self.storage_mode = False
|
||||
self.multicast_mode = False
|
||||
self.tsdb = None
|
||||
self.tsdb_module = False
|
||||
self.tsdb_class = False
|
||||
|
||||
def update_configs(self):
|
||||
# Initialize the key pair name
|
||||
@ -155,7 +161,7 @@ class KBConfig(object):
|
||||
|
||||
# If multicast mode, the number of receivers is specified in the multicast config instead.
|
||||
if self.multicast_mode:
|
||||
self.server_cfg['vms_per_network'] =\
|
||||
self.server_cfg['vms_per_network'] = \
|
||||
self.client_cfg['multicast_tool_configs']['receivers'][-1]
|
||||
|
||||
self.config_scale['server'] = self.server_cfg
|
||||
@ -169,6 +175,12 @@ class KBConfig(object):
|
||||
tc['rate'] = '0'
|
||||
if 'rate_iops' not in tc:
|
||||
tc['rate_iops'] = 0
|
||||
if not self.tsdb:
|
||||
self.tsdb = self.config_scale['tsdb']
|
||||
if not self.tsdb_module:
|
||||
self.tsdb_module = self.config_scale['tsdb']['module']
|
||||
if not self.tsdb_class:
|
||||
self.tsdb_class = self.config_scale['tsdb']['class']
|
||||
|
||||
def init_with_cli(self):
|
||||
self.storage_mode = CONF.storage
|
||||
|
||||
@ -17,6 +17,7 @@ from __init__ import __version__
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import datetime
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
@ -53,19 +54,22 @@ import tenant
|
||||
CONF = cfg.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KBVMCreationException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class KBFlavorCheckException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# flavor names to use
|
||||
FLAVOR_KB_PROXY = 'KB.proxy'
|
||||
FLAVOR_KB_CLIENT = 'KB.client'
|
||||
FLAVOR_KB_SERVER = 'KB.server'
|
||||
|
||||
class Kloud(object):
|
||||
|
||||
class Kloud(object):
|
||||
def __init__(self, scale_cfg, cred, reusing_tenants, vm_img,
|
||||
testing_side=False, storage_mode=False, multicast_mode=False):
|
||||
self.tenant_list = []
|
||||
@ -190,7 +194,6 @@ class Kloud(object):
|
||||
else:
|
||||
create_flavor(flavor_manager, FLAVOR_KB_SERVER, flavor_dict, extra_specs)
|
||||
|
||||
|
||||
def delete_resources(self):
|
||||
|
||||
if not self.reusing_tenants:
|
||||
@ -274,8 +277,6 @@ class Kloud(object):
|
||||
nc = instance.network.router.user.neutron_client
|
||||
base_network.disable_port_security(nc, instance.fixed_ip)
|
||||
|
||||
|
||||
|
||||
def create_vms(self, vm_creation_concurrency):
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=vm_creation_concurrency) as executor:
|
||||
@ -284,6 +285,7 @@ class Kloud(object):
|
||||
except Exception:
|
||||
self.exc_info = sys.exc_info()
|
||||
|
||||
|
||||
class KloudBuster(object):
|
||||
"""
|
||||
Creates resources on the cloud for loading up the cloud
|
||||
@ -296,7 +298,7 @@ class KloudBuster(object):
|
||||
|
||||
def __init__(self, server_cred, client_cred, server_cfg, client_cfg,
|
||||
topology, tenants_list, storage_mode=False, multicast_mode=False,
|
||||
interactive=False):
|
||||
interactive=False, tsdb_connector=None):
|
||||
# List of tenant objects to keep track of all tenants
|
||||
self.server_cred = server_cred
|
||||
self.client_cred = client_cred
|
||||
@ -305,6 +307,7 @@ class KloudBuster(object):
|
||||
self.storage_mode = storage_mode
|
||||
self.multicast_mode = multicast_mode
|
||||
self.interactive = interactive
|
||||
self.tsdb_connector = tsdb_connector
|
||||
|
||||
if topology and tenants_list:
|
||||
self.topology = None
|
||||
@ -665,6 +668,7 @@ class KloudBuster(object):
|
||||
self.print_provision_info()
|
||||
|
||||
def run_test(self, test_only=False):
|
||||
start_time = time.time()
|
||||
runlabel = None
|
||||
self.gen_metadata()
|
||||
self.kb_runner.config = self.client_cfg
|
||||
@ -682,7 +686,9 @@ class KloudBuster(object):
|
||||
for run_result in self.kb_runner.run(test_only, runlabel):
|
||||
if not self.multicast_mode or len(self.final_result['kb_result']) == 0:
|
||||
self.final_result['kb_result'].append(self.kb_runner.tool_result)
|
||||
|
||||
tsdb_result = self.tsdb_connector.get_results(start_time=start_time)
|
||||
if tsdb_result:
|
||||
self.final_result['tsdb'] = tsdb_result
|
||||
LOG.info('SUMMARY: %s' % self.final_result)
|
||||
if not self.interactive:
|
||||
break
|
||||
@ -724,7 +730,6 @@ class KloudBuster(object):
|
||||
self.kloud = None
|
||||
self.testing_kloud = None
|
||||
|
||||
|
||||
def dump_logs(self, offset=0):
|
||||
if not self.fp_logfile:
|
||||
return ""
|
||||
@ -858,6 +863,7 @@ class KloudBuster(object):
|
||||
|
||||
return quota_dict
|
||||
|
||||
|
||||
def create_html(hfp, template, task_re, is_config):
|
||||
for line in template:
|
||||
line = line.replace('[[result]]', task_re)
|
||||
@ -876,6 +882,7 @@ def create_html(hfp, template, task_re, is_config):
|
||||
url = 'file://' + os.path.abspath(CONF.html)
|
||||
webbrowser.open(url, new=2)
|
||||
|
||||
|
||||
def generate_charts(json_results, html_file_name, is_config):
|
||||
'''Save results in HTML format file.'''
|
||||
LOG.info('Saving results to HTML file: ' + html_file_name + '...')
|
||||
@ -895,6 +902,7 @@ def generate_charts(json_results, html_file_name, is_config):
|
||||
json.dumps(json_results, sort_keys=True),
|
||||
is_config)
|
||||
|
||||
|
||||
def main():
|
||||
cli_opts = [
|
||||
cfg.StrOpt("config",
|
||||
@ -1014,12 +1022,15 @@ def main():
|
||||
|
||||
# The KloudBuster class is just a wrapper class
|
||||
# levarages tenant and user class for resource creations and deletion
|
||||
tsdb_module = importlib.import_module(kb_config.tsdb_module)
|
||||
tsdb_connector = getattr(tsdb_module, kb_config.tsdb_class)(
|
||||
config=kb_config.tsdb)
|
||||
kloudbuster = KloudBuster(
|
||||
kb_config.cred_tested, kb_config.cred_testing,
|
||||
kb_config.server_cfg, kb_config.client_cfg,
|
||||
kb_config.topo_cfg, kb_config.tenants_list,
|
||||
storage_mode=CONF.storage, multicast_mode=CONF.multicast,
|
||||
interactive=CONF.interactive)
|
||||
interactive=CONF.interactive, tsdb_connector=tsdb_connector)
|
||||
if kloudbuster.check_and_upload_images():
|
||||
kloudbuster.run()
|
||||
|
||||
@ -1039,5 +1050,6 @@ def main():
|
||||
if CONF.html:
|
||||
generate_charts(kloudbuster.final_result, CONF.html, kb_config.config_scale)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
42
kloudbuster/prometheus.py
Normal file
42
kloudbuster/prometheus.py
Normal file
@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright 2018 Cisco Systems, Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import requests
|
||||
import time
|
||||
|
||||
|
||||
class Prometheus(object):
|
||||
def __init__(self, config):
|
||||
self.server_address = "http://{}:{}/api/v1/".format(config['server_ip'],
|
||||
config['server_port'])
|
||||
self.step_size = config['step_size']
|
||||
self.wait_time = config['wait_time']
|
||||
|
||||
def get_results(self, start_time, end_time=None):
|
||||
if not end_time:
|
||||
end_time = time.time()
|
||||
if end_time - start_time <= self.wait_time * 2:
|
||||
return None
|
||||
try:
|
||||
return requests.get(
|
||||
url="{}query_range?query=cpu_usage_system{{"
|
||||
"tag=%22ceph%22}}&start={}&end={}&step={}".format(self.server_address,
|
||||
start_time + self.wait_time,
|
||||
end_time - self.wait_time,
|
||||
self.step_size)).json()
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
print e
|
||||
return None
|
||||
31
kloudbuster/requirements.txt
Normal file
31
kloudbuster/requirements.txt
Normal file
@ -0,0 +1,31 @@
|
||||
# The order of packages is significant, because pip processes them in the order
|
||||
# of appearance. Changing the order has an impact on the overall integration
|
||||
# process, which may cause wedges in the gate later.
|
||||
|
||||
pytz>=2016.4
|
||||
pbr>=3.0.1
|
||||
Babel>=2.3.4
|
||||
|
||||
futures>=3.1.1
|
||||
python-cinderclient>=2.0.1
|
||||
python-glanceclient>=2.6.0
|
||||
python-openstackclient>=3.11.0
|
||||
python-neutronclient>=6.2.0
|
||||
# starting from 10.0.0, floating ip apis are removed from novaclient
|
||||
python-novaclient>=9.0.0,<10.0.0
|
||||
python-keystoneclient>=3.10.0
|
||||
attrdict>=2.0.0
|
||||
hdrhistogram>=0.5.2
|
||||
# ipaddress is required to get TLS working
|
||||
# otherwise certificates with numeric IP addresses in the ServerAltName field will fail
|
||||
ipaddress>= 1.0.16
|
||||
oslo.config>=4.1.1
|
||||
oslo.log>=3.26.1
|
||||
pecan>=1.2.1
|
||||
redis>=2.10.5
|
||||
tabulate>=0.7.7
|
||||
pyyaml>=3.12
|
||||
requests
|
||||
|
||||
# Workaround for pip install failed on RHEL/CentOS
|
||||
functools32>=3.2.3
|
||||
24
kloudbuster/tsdb.py
Normal file
24
kloudbuster/tsdb.py
Normal file
@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright 2018 Cisco Systems, Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import time
|
||||
|
||||
|
||||
class TSDB(object):
|
||||
def __init__(self, config):
|
||||
pass
|
||||
|
||||
def get_results(self, start_time, end_time=time.time()):
|
||||
pass
|
||||
Loading…
x
Reference in New Issue
Block a user