Integrating KloudBuster with DIB

1. Integrating KloudBuster with DIB;
2. Auto upload images to Nova if image not present;
3. Change default login username to cloud-user;

Change-Id: Ib47a6827a14aa0cb85fb97e2aa708b016fc2d6d8
This commit is contained in:
Yichen Wang 2015-06-04 19:00:25 -07:00
parent 21e6b0ef92
commit 8b50d685c6
12 changed files with 105 additions and 236 deletions

View File

@ -3,7 +3,7 @@
# Name of the image to use for all test VMs (client, server and proxy)
# The image name must exist in OpenStack and must be built with the appropriate
# packages
image_name: 'Scale Image v8'
image_name: 'KloudBuster Image'
# Config options common to client and server side
keystone_admin_role: "admin"
@ -12,7 +12,7 @@ keystone_admin_role: "admin"
cleanup_resources: True
# VM creation concurrency
vm_creation_concurrency: 10
vm_creation_concurrency: 5
#
# ssh access to the test VMs launched by kloudbuster is not required
@ -52,13 +52,13 @@ server:
networks_per_router: 1
# Number of VM instances to be created within the context of each Network
vms_per_network: 2
vms_per_network: 1
# Number of security groups per network
secgroups_per_network: 1
# Assign floating IP for every VM
use_floatingip: False
use_floatingip: True
# Placement hint
# Availability zone to use for servers in the server cloud
@ -72,7 +72,7 @@ server:
# CLIENT SIDE CONFIG OPTIONS
client:
# Assign floating IP for every VM
use_floatingip: False
use_floatingip: True
# Flavor to use for the test images
flavor:
@ -97,10 +97,10 @@ client:
# Tooling
tp_tool:
name: 'nuttcp'
dest_path: '/var/tmp/nuttcp-7.3.2'
dest_path: '/usr/bin/nuttcp'
http_tool:
name: 'wrk'
dest_path: '/var/tmp/wrk2-3.1.1'
dest_path: '/usr/local/bin/wrk2'
# HTTP tool specific configs (per VM)
http_tool_configs:

View File

@ -1,4 +1,6 @@
# script for kloudbuster
# Tune the Linux kernel for scale test
echo "* soft nofile 102400" >> /etc/security/limits.conf
echo "* hard nofile 102400" >> /etc/security/limits.conf
@ -33,12 +35,28 @@ sysctl -p
update-rc.d -f redis-server remove
update-rc.d -f nginx remove
# Change permission to 777 for kb_test
chmod -R 777 /kb_test
# Generate a 32K HTML files
mkdir -p /data/www
cd /data/www/
dd if=/dev/zero of=index.html bs=32K count=1
chmod -R 777 /data
cd -
# redis server should listen on all interfaces
sed -i "s/127.0.0.1/0.0.0.0/g" /etc/redis/redis.conf
# if started nginx should be allowed to open more file descriptors
sed -i 's/start-stop-daemon\ --start/ulimit\ \-n\ 102400\n\t\0/g' /etc/init.d/nginx
# Auto start the KloudBuster Agent, with user-data
sed -i "s/^exit\s0/cd \/kb_test\n\0/g" /etc/rc.local
sed -i "s/^exit\s0/wget http\:\/\/169.254.169.254\/latest\/user-data\n\0/g" /etc/rc.local
sed -i "s/^exit\s0/python \/kb_test\/kb_vm_agent.py \&\n\0/g" /etc/rc.local
# ======
# Client
# ======
@ -50,7 +68,7 @@ pip install redis
git clone git://github.com/giltene/wrk2.git
cd wrk2
make
mv wrk /usr/local/bin
mv wrk /usr/local/bin/wrk2
cd ..
rm -rf wrk2
@ -58,5 +76,5 @@ rm -rf wrk2
apt-get -y --purge remove git
apt-get -y --purge remove python-pip
apt-get -y --purge remove build-essential
apt-get -y autoremove
apt-get -y --purge autoremove
apt-get -y autoclean

View File

@ -0,0 +1,33 @@
#!/usr/bin/env python
import yaml
cloudcfg = "/etc/cloud/cloud.cfg"
user = "cloud-user"
with open(cloudcfg) as f:
cfg = yaml.load(f)
try:
if cfg['system_info']['default_user']['name']:
synver = "2"
except KeyError:
synver = "1"
if synver == "1":
if cfg['user'] == user:
print("No change needed")
exit()
else:
cfg['user'] = user
elif synver == "2":
if cfg['system_info']['default_user']['name'] == user:
print("No change needed")
exit()
else:
# Change the user to cloud-user
cfg['system_info']['default_user']['name'] = user
cfg['system_info']['default_user']['gecos'] = "Cloud User"
print cfg['system_info']['default_user']['name']
with open(cloudcfg, "w") as f:
yaml.dump(cfg, f, default_flow_style=False)

View File

@ -1,36 +1,11 @@
user nginx;
worker_processes 8;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
user www-data;
worker_processes 8;
pid /run/nginx.pid;
events {
worker_connections 2048;
worker_connections 2048;
}
# http {
# include /etc/nginx/mime.types;
# default_type application/octet-stream;
#
# log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#
# access_log /var/log/nginx/access.log main;
#
# sendfile on;
# #tcp_nopush on;
#
# keepalive_timeout 65;
#
# #gzip on;
#
# include /etc/nginx/conf.d/*.conf;
# }
http {
server {
location / {

View File

@ -78,7 +78,7 @@ class KB_Instance(object):
rate_limit, duration, timeout, connection_type):
if not rate_limit:
rate_limit = 65535
cmd = '%s -t%d -c%d -R%d -d%ds --timeout %ds --latency --s kb.lua %s' % \
cmd = '%s -t%d -c%d -R%d -d%ds --timeout %ds --latency --s /kb_test/kb_wrk2.lua %s' % \
(dest_path, threads, connections, rate_limit, duration, timeout, target_url)
return cmd
@ -203,7 +203,7 @@ def start_redis_server():
return exec_command(cmd)
def start_nuttcp_server():
cmd = ['/var/tmp/nuttcp-7.3.2', '-P5002', '-S', '--single-threaded']
cmd = ['/usr/bin/nuttcp', '-P5002', '-S', '--single-threaded']
return exec_command(cmd)
def start_nginx_server():
@ -212,7 +212,7 @@ def start_nginx_server():
if __name__ == "__main__":
try:
f = open('/var/tmp/user-data', 'r')
f = open('user-data', 'r')
user_data = eval(f.read())
except Exception as e:
# TODO(Logging on Agent)

View File

@ -1,101 +0,0 @@
#!/bin/sh
# Copyright 2015 Cisco Systems, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
##############################################################################
# The DNS Server
KB_DNS_SERVER='8.8.8.8'
# Image Version
KB_IMAGE_VERSION='8'
# The base image used to build the snapshot
# Note: Must be a Debian-based Image
KB_BASE_IMAGE_NAME='Ubuntu Server 14.04'
# The URL for downloading the base image
KB_BASE_IMAGE_URL='https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img'
# Save the snapshot image to local host
KB_SAVE_SNAPSHOT_IMAGE='yes'
##############################################################################
# ==================
# Creating Resources
# ==================
KB_SNAPSHOT_IMG_NAME="Scale Image v$KB_IMAGE_VERSION"
KB_SNAPSHOT_IMG_FILENAME="scale_image_v$KB_IMAGE_VERSION.qcow2"
KB_EXTERNAL_NET=`neutron net-list -D -c 'name' -c 'router:external' | grep True -m1 | cut -d'|' -f2 | xargs`
rm -f /var/tmp/kb_image_key /var/tmp/kb_image_key.pub
ssh-keygen -b 1024 -C "KloudBuster Image" -t rsa -f /var/tmp/kb_image_key -N ""
nova secgroup-create kb_secgroup "Temp Security Group for creating KloudBuster image"
neutron security-group-rule-create kb_secgroup --direction ingress
neutron net-create kb_net
neutron subnet-create kb_net 192.168.1.0/24 --name kb_subnet --gateway 192.168.1.1 --allocation-pool start=192.168.1.100,end=192.168.1.200 --dns-nameserver $KB_DNS_SERVER
neutron router-create kb_router
neutron router-gateway-set kb_router $KB_EXTERNAL_NET
neutron router-interface-add kb_router kb_subnet
nova keypair-add --pub-key /var/tmp/kb_image_key.pub kb_image_key
glance image-list | grep "\b$KB_BASE_IMAGE_NAME\b"
if [ $? -ne 0 ]; then
glance image-create --name="$KB_BASE_IMAGE_NAME" --disk-format=qcow2 --container-format=bare --is-public True --copy-from $KB_BASE_IMAGE_URL
fi
KB_NET_ID=`neutron net-list | grep 'kb_net' | cut -d'|' -f2 | xargs`
KB_FLOATING_IP_ID=`neutron floatingip-create $KB_EXTERNAL_NET | grep '\bid\b' | cut -d'|' -f3 | xargs`
KB_FLOATING_IP=`neutron floatingip-list | grep $KB_FLOATING_IP_ID | cut -d'|' -f4 | xargs`
nova boot kb_scale_instance --flavor m1.small --image "$KB_BASE_IMAGE_NAME" --key-name kb_image_key --security-group kb_secgroup --nic net-id=$KB_NET_ID --poll
KB_INSTANCE_IP=`nova list | grep 'kb_scale_instance' | cut -d'=' -f2 | sed 's/\s*|//g'`
KB_PORT_ID=`neutron port-list | grep $KB_INSTANCE_IP | cut -d'|' -f2 | xargs`
neutron floatingip-associate $KB_FLOATING_IP_ID $KB_PORT_ID
# Running scripts to build image
while true; do
ssh ubuntu@$KB_FLOATING_IP -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i /var/tmp/kb_image_key echo "Connected"
if [ $? -eq 0 ]; then break; fi
sleep 5
done
ssh -o StrictHostKeyChecking=no -i /var/tmp/kb_image_key ubuntu@$KB_FLOATING_IP 'bash -s' < scripts
# ===============
# Create Snapshot
# ===============
nova stop kb_scale_instance
while true; do
sleep 1
nova list | grep kb_scale_instance | grep SHUTOFF
if [ $? -eq 0 ]; then
break;
fi
done
nova image-create --poll kb_scale_instance "$KB_SNAPSHOT_IMG_NAME"
if [ "$KB_SAVE_SNAPSHOT_IMAGE" = "yes" ]; then
echo "Saving snapshot to $KB_SNAPSHOT_IMG_FILENAME..."
glance image-download "$KB_SNAPSHOT_IMG_NAME" --file $KB_SNAPSHOT_IMG_FILENAME
fi
# =======
# Cleanup
# =======
nova delete kb_scale_instance
while true; do
sleep 1
nova list | grep kb_scale_instance
if [ $? -ne 0 ]; then
break;
fi
done
neutron floatingip-delete $KB_FLOATING_IP_ID
neutron router-interface-delete kb_router kb_subnet
neutron router-gateway-clear kb_router
neutron router-delete kb_router
neutron net-delete kb_net
nova secgroup-delete kb_secgroup
nova keypair-delete kb_image_key
rm -f /var/tmp/kb_image_key /var/tmp/kb_image_key.pub

View File

@ -1,90 +0,0 @@
#!bin/sh
# Copyright 2015 Cisco Systems, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
sudo bash -c 'echo "* soft nofile 102400" >> /etc/security/limits.conf'
sudo bash -c 'echo "* hard nofile 102400" >> /etc/security/limits.conf'
sudo bash -c 'echo "fs.file-max=6553550" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.core.wmem_max=8388608" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.core.wmem_default=8388608" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.core.rmem_max=33554432" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.core.rmem_default=33554432" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.core.netdev_max_backlog=100000" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.icmp_ratelimit=0" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.tcp_tw_recycle=1" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.tcp_tw_reuse=1" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.tcp_max_tw_buckets=65536" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.tcp_fin_timeout=15" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.tcp_max_syn_backlog=65536" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.tcp_syncookies=1" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.neigh.default.gc_thresh1=4096" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.neigh.default.gc_thresh2=4096" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.neigh.default.gc_thresh3=4096" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.conf.all.rp_filter=0" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.conf.all.arp_filter=0" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.conf.default.rp_filter=0" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.conf.default.arp_filter=0" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.conf.eth0.rp_filter=0" >> /etc/sysctl.conf'
sudo bash -c 'echo "net.ipv4.conf.eth0.arp_filter=0" >> /etc/sysctl.conf'
sudo sysctl -p
cd /var/tmp
wget http://nginx.org/keys/nginx_signing.key
sudo apt-key add nginx_signing.key
sudo bash -c "echo 'deb http://nginx.org/packages/ubuntu/ trusty nginx' >> /etc/apt/sources.list"
sudo apt-get update
sudo apt-get -y install nginx redis-server
sudo update-rc.d -f redis-server remove
sudo update-rc.d -f nginx remove
wget http://172.29.172.152/downloads/scale_image/nginx.conf
sudo mv nginx.conf /etc/nginx/nginx.conf
sudo mkdir /data
sudo mkdir /data/www
sudo chmod -R 777 /data
cd /data/www
wget http://172.29.172.152/downloads/scale_image/index.tar.gz
tar xzf index.tar.gz
rm index.tar.gz
cd /var/tmp
wget http://172.29.172.152/downloads/scale_image/nuttcp-7.3.2
chmod +x nuttcp-7.3.2
sudo sed -i "s/127.0.0.1/0.0.0.0/g" /etc/redis/redis.conf
sudo sed -i 's/start-stop-daemon\ --start/ulimit\ \-n\ 102400\n\ \ \ \ \0/g' /etc/init.d/nginx
sudo sed -i "s/^exit\s0/cd \/var\/tmp\n\0/g" /etc/rc.local
sudo sed -i "s/^exit\s0/wget http\:\/\/169.254.169.254\/latest\/user-data\n\0/g" /etc/rc.local
sudo sed -i "s/^exit\s0/python \/var\/tmp\/kb_vm_agent.py \&\n\0/g" /etc/rc.local
sudo sed -i "s/^exit\s0/cd -\n\n\0/g" /etc/rc.local
# ======
# Client
# ======
cd /var/tmp
# sudo apt-get -y install libgo5
sudo apt-get clean
wget http://172.29.172.152/downloads/scale_image/redis-2.10.3.tar.gz
tar xzf redis-2.10.3.tar.gz
cd redis-2.10.3
sudo python setup.py install
cd -
sudo rm -rf redis-2.10.3
rm -f redis-2.10.3.tar.gz
wget http://172.29.172.152/downloads/scale_image/wrk-4.0.1
chmod +x wrk-4.0.1
wget http://172.29.172.152/downloads/scale_image/wrk2-3.1.1
chmod +x wrk2-3.1.1
# wget http://172.29.172.152/downloads/scale_image/gobench
# chmod +x gobench
wget http://172.29.172.152/downloads/scale_image/kb_vm_agent.py
wget http://172.29.172.152/downloads/scale_image/kb.lua
sync
history -cw

View File

@ -1 +0,0 @@
dib/elements/kloudbuster/static/kb_test/kb.lua

View File

@ -55,8 +55,6 @@ class KBRunner(object):
def setup_redis(self, redis_server, redis_server_port=6379, timeout=120):
LOG.info("Setting up redis connection pool...")
# For now, the redis server is not in the scope of Kloud Buster, which has to be
# pre-configured before executing Kloud Buster.
connection_pool = redis.ConnectionPool(
host=redis_server, port=redis_server_port, db=0)
@ -157,7 +155,7 @@ class KBRunner(object):
raise KBVMUpException()
self.send_cmd('ACK', None, None)
def setup_static_route(self, timeout=10):
def setup_static_route(self, timeout=30):
func = {'cmd': 'setup_static_route'}
self.send_cmd('EXEC', 'http', func)
cnt_succ = self.polling_vms(timeout)[0]

1
scale/kb_wrk2.lua Symbolic link
View File

@ -0,0 +1 @@
dib/elements/kloudbuster/static/kb_test/kb_wrk2.lua

View File

@ -22,6 +22,7 @@ import traceback
import base_compute
import base_network
import configure
from glanceclient.v2 import client as glanceclient
from kb_runner import KBRunner
from kb_scheduler import KBScheduler
from keystoneclient.v2_0 import client as keystoneclient
@ -60,6 +61,35 @@ def create_keystone_client(admin_creds):
creds = admin_creds.get_credentials()
return (keystoneclient.Client(**creds), creds['auth_url'])
def check_and_upload_images(cred, cred_testing, server_img_name, client_img_name):
keystone_list = [create_keystone_client(cred)[0], create_keystone_client(cred_testing)[0]]
keystone_dict = dict(zip(['Server kloud', 'Client kloud'], keystone_list))
img_name_dict = dict(zip(['Server kloud', 'Client kloud'], [server_img_name, client_img_name]))
for kloud, keystone in keystone_dict.items():
img_found = False
glance_endpoint = keystone.service_catalog.url_for(
service_type='image', endpoint_type='publicURL')
glance_client = glanceclient.Client(glance_endpoint, token=keystone.auth_token)
for img in glance_client.images.list():
if img['name'] == img_name_dict[kloud]:
img_found = True
break
if not img_found:
# Trying upload images
LOG.info("Image is not found in %s, trying to upload..." % (kloud))
if not os.path.exists('dib/kloudbuster.qcow2'):
LOG.error("Image file dib/kloudbuster.qcow2 is not present, please refer "
"to dib/README.rst for how to build image for KloudBuster.")
return False
with open('dib/kloudbuster.qcow2') as fimage:
image = glance_client.images.create(name=img_name_dict[kloud],
disk_format="qcow2",
container_format="bare")
glance_client.images.upload(image['id'], fimage)
return True
class Kloud(object):
def __init__(self, scale_cfg, admin_creds, testing_side=False):
self.cred = admin_creds
@ -372,6 +402,7 @@ hardcoded_client_cfg = {
'secgroups_per_network': 1
}
if __name__ == '__main__':
# The default configuration file for KloudBuster
default_cfg_file = get_absolute_path_for_file("cfg.scale.yaml")
@ -459,6 +490,11 @@ if __name__ == '__main__':
# There is an additional VM in client kloud as a proxy node
client_side_cfg['vms_per_network'] = get_total_vm_count(server_side_cfg) + 1
image_check = check_and_upload_images(cred, cred_testing, server_side_cfg.image_name,
client_side_cfg.image_name)
if not image_check:
sys.exit(1)
# The KloudBuster class is just a wrapper class
# levarages tenant and user class for resource creations and
# deletion