From c8727d1431e94602b8d1bd2cf64f8df630fe0b67 Mon Sep 17 00:00:00 2001 From: Dmitry Sutyagin Date: Thu, 23 Feb 2017 15:00:24 -0800 Subject: [PATCH] Fix: UnicodeDecodeError, broken spec; add SPT - Only convert outputs/errors to unicode if printing required, use 'replace' error handling to avoid errors. Write original byte str to output files to avoid double conversion. - Fix spec broken with 1.26.6. - Add simplified-performance-testing config & rq files, scripts. Can be used after deployment to get a few benchmark values. SPT_parser.sh can be used to parse results. Change-Id: I2aacfdbd9574ff737a6b88a29d87ae56abd03e46 --- specs/python-timmy.spec | 9 +- timmy/env.py | 2 +- timmy/nodes.py | 17 +- timmy/tools.py | 27 +- .../simplified-performance-testing/SPT.yaml | 13 + .../SPT_parser.sh | 154 ++++++++ .../config.yaml | 7 + .../simplified-performance-testing/readme.txt | 2 + .../scripts/cinder-VM-volume-write.sh | 197 ++++++++++ .../scripts/glance-1-image-create.sh | 11 + .../scripts/glance-1-image-download.sh | 8 + .../scripts/iperf-client.sh | 28 ++ .../scripts/iperf-server-start.sh | 23 ++ .../scripts/iperf-server-stop.sh | 18 + .../scripts/network-VM-to-VM-iperf-tests.sh | 350 ++++++++++++++++++ 15 files changed, 840 insertions(+), 26 deletions(-) create mode 100644 timmy_data/simplified-performance-testing/SPT.yaml create mode 100755 timmy_data/simplified-performance-testing/SPT_parser.sh create mode 100644 timmy_data/simplified-performance-testing/config.yaml create mode 100644 timmy_data/simplified-performance-testing/readme.txt create mode 100644 timmy_data/simplified-performance-testing/scripts/cinder-VM-volume-write.sh create mode 100644 timmy_data/simplified-performance-testing/scripts/glance-1-image-create.sh create mode 100644 timmy_data/simplified-performance-testing/scripts/glance-1-image-download.sh create mode 100644 timmy_data/simplified-performance-testing/scripts/iperf-client.sh create mode 100644 timmy_data/simplified-performance-testing/scripts/iperf-server-start.sh create mode 100644 timmy_data/simplified-performance-testing/scripts/iperf-server-stop.sh create mode 100644 timmy_data/simplified-performance-testing/scripts/network-VM-to-VM-iperf-tests.sh diff --git a/specs/python-timmy.spec b/specs/python-timmy.spec index 6bcb006..c92b073 100644 --- a/specs/python-timmy.spec +++ b/specs/python-timmy.spec @@ -4,7 +4,7 @@ %global pypi_name timmy Name: python-%{pypi_name} -Version: 1.26.6 +Version: 1.26.7 Release: 1%{?dist}~mos0 Summary: Log collector tool for OpenStack Fuel @@ -107,7 +107,12 @@ popd %changelog -* Thu Jan 19 2016 Dmitry Sutyagin - 1.26.6 +* Thu Feb 23 2017 Dmitry Sutyagin - 1.26.7 +- Fix: UnicodeDecodeError, broken spec; add SPT +- Update documentation to match timmy 1.26.6 code +- Fix: CLI docs + +* Thu Jan 19 2017 Dmitry Sutyagin - 1.26.6 - Add yum.repos.d directory to snapshot - Change: print only summary for analysis module if all nodes are ok - Replace yaml.load() with yaml.safe_load() diff --git a/timmy/env.py b/timmy/env.py index 08e4490..d53da5b 100644 --- a/timmy/env.py +++ b/timmy/env.py @@ -16,7 +16,7 @@ # under the License. project_name = 'timmy' -version = '1.26.6' +version = '1.26.7' if __name__ == '__main__': import sys diff --git a/timmy/nodes.py b/timmy/nodes.py index cc83d07..73c867a 100644 --- a/timmy/nodes.py +++ b/timmy/nodes.py @@ -252,7 +252,7 @@ class Node(object): c[cmd], errs, ok_codes) try: with open(dfile, 'w') as df: - df.write(outs.encode('utf-8')) + df.write(outs) except IOError: self.logger.error("can't write to file %s" % dfile) @@ -260,7 +260,7 @@ class Node(object): try: with open(errf, 'w') as ef: ef.write('exitcode: %s\n' % code) - ef.write(errs.encode('utf-8')) + ef.write(errs) except IOError: self.logger.error("can't write to file %s" % errf) @@ -281,7 +281,7 @@ class Node(object): errs, ok_codes) try: with open(param['output_path'], 'w') as df: - df.write(outs.encode('utf-8')) + df.write(outs) except IOError: self.logger.error("can't write to file %s" % param['output_path']) @@ -289,14 +289,14 @@ class Node(object): try: with open(param['stderr_path'], 'w') as ef: ef.write('exitcode: %s\n' % code) - ef.write(errs.encode('utf-8')) + ef.write(errs) except IOError: self.logger.error("can't write to file %s" % param['stderr_path']) return mapcmds, self.mapscr def exec_simple_cmd(self, cmd, timeout=15, infile=None, outfile=None, - fake=False, ok_codes=None, input=None, decode=True): + fake=False, ok_codes=None, input=None): self.logger.info('%s, exec: %s' % (self.repr, cmd)) if not fake: outs, errs, code = tools.ssh_node(ip=self.ip, @@ -306,7 +306,6 @@ class Node(object): timeout=timeout, outputfile=outfile, ok_codes=ok_codes, - decode=decode, input=input, prefix=self.prefix) self.check_code(code, 'exec_simple_cmd', cmd, errs, ok_codes) @@ -515,9 +514,10 @@ class Node(object): def check_code(self, code, func_name, cmd, err, ok_codes=None): if code: if not ok_codes or code not in ok_codes: + p_err = unicode(err, 'utf-8', 'replace').rstrip('\n') self.logger.warning("%s: func: %s: " "cmd: '%s' exited %d, error: %s" % - (self.repr, func_name, cmd, code, err)) + (self.repr, func_name, cmd, code, p_err)) return False return True @@ -907,8 +907,7 @@ class NodeManager(object): 'timeout': timeout, 'outfile': node.archivelogsfile, 'input': input, - 'ok_codes': [0, 1], - 'decode': False} + 'ok_codes': [0, 1]} run_items.append(tools.RunItem(target=node.exec_simple_cmd, args=args)) tools.run_batch(run_items, self.logs_maxthreads) diff --git a/timmy/tools.py b/timmy/tools.py index 6169118..c14c7f1 100644 --- a/timmy/tools.py +++ b/timmy/tools.py @@ -282,7 +282,7 @@ def mdir(directory): sys.exit(110) -def launch_cmd(cmd, timeout, input=None, ok_codes=None, decode=True): +def launch_cmd(cmd, timeout, input=None, ok_codes=None): def _timeout_terminate(pid): try: os.kill(pid, 15) @@ -303,26 +303,25 @@ def launch_cmd(cmd, timeout, input=None, ok_codes=None, decode=True): timeout_killer = threading.Timer(timeout, _timeout_terminate, [p.pid]) timeout_killer.start() outs, errs = p.communicate(input=input) - errs = errs.rstrip('\n') - if decode: - outs = outs.decode('utf-8') - errs = errs.decode('utf-8') finally: if timeout_killer: timeout_killer.cancel() - input = input.decode('utf-8') if input else None - logger.debug(('___command: %s\n' - '_______pid: %s\n' - '_exit_code: %s\n' - '_____stdin: %s\n' - '____stderr: %s') % (cmd, p.pid, p.returncode, input, - errs)) + if logger.isEnabledFor(logging.DEBUG): + # p_out = unicode(outs, 'utf-8', 'replace') + p_err = unicode(errs, 'utf-8', 'replace').rstrip('\n') + p_inp = unicode(input, 'utf-8', 'replace') if input else None + logger.debug(('___command: %s\n' + '_______pid: %s\n' + '_exit_code: %s\n' + '_____stdin: %s\n' + '____stderr: %s') % (cmd, p.pid, p.returncode, p_inp, + p_err)) return outs, errs, p.returncode def ssh_node(ip, command='', ssh_opts=None, env_vars=None, timeout=15, filename=None, inputfile=None, outputfile=None, - ok_codes=None, input=None, prefix=None, decode=True): + ok_codes=None, input=None, prefix=None): if not ssh_opts: ssh_opts = '' if not env_vars: @@ -354,7 +353,7 @@ def ssh_node(ip, command='', ssh_opts=None, env_vars=None, timeout=15, "trap 'kill $pid' 2; echo -n \"$input\" | xxd -r -p | " + cmd + ' &:; pid=$!; wait $!') return launch_cmd(cmd, timeout, input=input, - ok_codes=ok_codes, decode=decode) + ok_codes=ok_codes) def get_files_rsync(ip, data, ssh_opts, rsync_opts, dpath, timeout=15): diff --git a/timmy_data/simplified-performance-testing/SPT.yaml b/timmy_data/simplified-performance-testing/SPT.yaml new file mode 100644 index 0000000..08403b9 --- /dev/null +++ b/timmy_data/simplified-performance-testing/SPT.yaml @@ -0,0 +1,13 @@ +scripts: + once_by_roles: + controller: + - 'glance-1-image-create.sh' + - 'glance-2-image-download.sh' + - {'cinder-VM-volume-write.sh': 'SPT_FLOATING_NET="admin_floating_net" SPT_FLAVOR="m1.small" SPT_IMAGE="xenial" SPT_VM_USER="ubuntu" DD_OPTIONS="oflag=direct" SPT_VM_COOLDOWN="120" DD_TIMEOUT="10" VM_BOOT_TIMEOUT_MINUTES="5"'} + - {'network-VM-to-VM-iperf-tests.sh': 'SPT_FLOATING_NET="admin_floating_net" SPT_FLAVOR="m1.small" SPT_IMAGE="xenial" SPT_VM_USER="ubuntu" SPT_VM_COOLDOWN="120" VM_BOOT_TIMEOUT_MINUTES="5" SPT_AVAILABILITY_ZONE="nova"'} +scripts_all_pairs: + __default: + - server_start: {'iperf-server-start.sh': 'SPT_IPERF_PORT=65432'} + server_stop: 'iperf-server-stop.sh' + client: {'iperf-client.sh': 'SPT_IPERF_PORT=65432'} + network: 'management' diff --git a/timmy_data/simplified-performance-testing/SPT_parser.sh b/timmy_data/simplified-performance-testing/SPT_parser.sh new file mode 100755 index 0000000..93d264b --- /dev/null +++ b/timmy_data/simplified-performance-testing/SPT_parser.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +results=${1:-"/tmp/timmy/info"} +glance_create="glance-1-image-create.sh" +glance_download="glance-2-image-download.sh" +cinder_volume="cinder-VM-volume-write.sh" +iperf_vm="network-VM-to-VM-iperf-tests.sh" +iperf_host="iperf-client.sh" +res_glance_create=`find $results -name $glance_create` +res_glance_download=`find $results -name $glance_download` +res_cinder=`find $results -name $cinder_volume` +res_iperf_vm=`find $results -name $iperf_vm` +res_iperf_host="$(find $results -name "${iperf_host}*")" +res_iperf_node_dir="$(find $results -name "client" -type d)" +[ -n "$res_iperf_node_dir" ] && res_iperf_node="$(ls $res_iperf_node_dir)" || res_iperf_node="" + +function print_result() { + [ -n "$2" ] && printf "$2" || printf "n/a" + echo -e "\t<-- $1" +} + +function a() { + [ -n "$1" ] && printf "$1\t" || printf "n/a\t" +} + +function b() { + word=`echo $1 | grep -o '[a-zA-Z/]\+'` + number=`echo $1 | grep -o '[0-9.]\+'` + python -c ' +import sys +word = sys.argv[1] +number = sys.argv[2] +if word == "Gbits/sec": + sys.stdout.write(str(int(float(number)*1000000000))) +elif word == "Mbits/sec": + sys.stdout.write(str(int(float(number)*1000000))) +elif word == "Kbits/sec": + sys.stdout.write(str(int(float(number)*1000))) +elif word == "bits/sec": + sys.stdout.write(str(int(number))) +' $word $number 2> /dev/null +} + +function c() { + echo $1 | python -c 'import sys; print(float(sys.stdin.read())/1000000000)' 2> /dev/null +} + +function d() { + word=`echo $1 | grep -o '[a-zA-Z/]\+'` + number=`echo $1 | grep -o '[0-9.]\+'` + python -c ' +import sys +word = sys.argv[1] +number = sys.argv[2] +if word == "GB/s": + sys.stdout.write(str(int(float(number)*1000))) +elif word == "MB/s": + sys.stdout.write(str(float(number))) +elif word == "kB/s": + sys.stdout.write(str(int(float(number)/1000))) +' $word $number 2> /dev/null +} + +if [ -n "$res_glance_create" ] +then + res_gc=`head -n 4 $res_glance_create | grep '^[0-9.]\+$' | python -c 'import sys; print("%.2f" % (4000/float(sys.stdin.read(),)))'` + res_gc=${res_gc}'MB/s' +fi +print_result "Glance upload" $res_gc + +if [ -n "$res_glance_download" ] +then + res_gd=`head -n 1 $res_glance_download | grep '^[0-9.]\+$' | python -c 'import sys; print("%.2f" % (4000/float(sys.stdin.read(),)))'` + res_gd=${res_gd}'MB/s' +fi +print_result "Glance download" $res_gd + +if [ -n "$res_cinder" ] +then + res_4k=`grep 'DD_TEST_1' $res_cinder | grep copied | rev | awk '{print $1$2}' | rev` + res_1m=`grep 'DD_TEST_2' $res_cinder | grep copied | rev | awk '{print $1$2}' | rev` + res_1g=`grep 'DD_TEST_3' $res_cinder | grep copied | rev | awk '{print $1$2}' | rev` +fi +print_result "Block Storage Write 4k" $res_4k +print_result "Block Storage Write 1M" $res_1m +print_result "Block Storage Write 1G" $res_1g + +if [ -n "$res_iperf_host" ] +then + res_t1="" + res_t10="" + for i in $res_iperf_host + do + res_t1="$(echo "$res_t1"; b $(head -n 7 $i | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'))" + res_t1="$(echo "$res_t1" | grep -v '^$' | sort -n | tail -n 1)" + res_t10="$(echo "$res_t10"; b $(head -n 33 $i | tail -n 1 | grep '^\[SUM.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'))" + res_t10="$(echo "$res_t10" | grep -v '^$' | sort -n | tail -n 1)" + done +fi +res_t1=`echo "$res_t1" | python -c 'import sys; print(float(sys.stdin.read())/1000000000)' 2> /dev/null` +res_t10=`echo "$res_t10" | python -c 'import sys; print(float(sys.stdin.read())/1000000000)' 2> /dev/null` +print_result "HW to HW (best)" $res_t1 +print_result "HW to HW (best) - 10 Threads" $res_t10 + +if [ -n "$res_iperf_vm" ] +then + res1=`head -n 8 $res_iperf_vm | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'` + res2=`head -n 17 $res_iperf_vm | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'` + res3=`head -n 44 $res_iperf_vm | tail -n 1 | grep '^\[SUM.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'` + res4=`head -n 52 $res_iperf_vm | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'` + res5=`head -n 61 $res_iperf_vm | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'` +fi +print_result "VM to VM - VMs on same node - via Private IP - 1 thread" $res1 +print_result "VM to VM - VMs on different HW nodes - via Private IP - 1 thread" $res2 +print_result "VM to VM - VMs on different HW nodes - via Private IP - MILTI 10 thread" $res3 +print_result "VM to VM - via Floating IP and VMs are on different nodes - 1 thread" $res4 +print_result "VM to VM - diff nodes, VMs connected to separate networks connected by vRouter - via Private IP - 1 thread" $res5 + +[ -n "$res_iperf_node" ] && nnum="$(wc -l <<< "$res_iperf_node")" && print_result "Number of nodes (during iperf HW to HW tests)" $nnum + +if [ -n "$res_iperf_host" ] +then + res_mt1="" + res_mt10="" + for i in $res_iperf_host + do + res_mt1="$(echo "$res_mt1"; b $(head -n 7 $i | tail -n 1 | grep '^\[.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'))" + res_mt1="$(echo "$res_mt1" | grep -v '^$' | sort -n | head -n 1)" + res_mt10="$(echo "$res_mt10"; b $(head -n 33 $i | tail -n 1 | grep '^\[SUM.*sec' | rev | awk '{print $1$2}' | rev | sed 's/ //g'))" + res_mt10="$(echo "$res_mt10" | grep -v '^$' | sort -n | head -n 1)" + done +fi +res_mt1=`echo "$res_mt1" | python -c 'import sys; print(float(sys.stdin.read())/1000000000)' 2> /dev/null` +res_mt10=`echo "$res_mt10" | python -c 'import sys; print(float(sys.stdin.read())/1000000000)' 2> /dev/null` +print_result "HW to HW (worst)" $res_mt1 +print_result "HW to HW (worst) - 10 Threads" $res_mt10 + +echo "--------------------------" +a `d $res_gc` +a `d $res_gd` +a `d $res_4k` +a `d $res_1m` +a `d $res_1g` +a $res_t1 # HW to HW +a $res_t10 # HW to HW +a $(c `b $res1`) +a $(c `b $res2`) +a $(c `b $res3`) +a $(c `b $res4`) +a $(c `b $res5`) +a $nnum +a $res_mt1 # HW to HW max concurrency +a $res_mt10 # HW to HW max concurrency +echo diff --git a/timmy_data/simplified-performance-testing/config.yaml b/timmy_data/simplified-performance-testing/config.yaml new file mode 100644 index 0000000..48c9832 --- /dev/null +++ b/timmy_data/simplified-performance-testing/config.yaml @@ -0,0 +1,7 @@ +rqfile: + - file: simplified-performance-testing/SPT.yaml + default: false +rqdir: 'simplified-performance-testing' +timeout: 3600 +soft_filter: + no_roles: ['fuel'] diff --git a/timmy_data/simplified-performance-testing/readme.txt b/timmy_data/simplified-performance-testing/readme.txt new file mode 100644 index 0000000..3104925 --- /dev/null +++ b/timmy_data/simplified-performance-testing/readme.txt @@ -0,0 +1,2 @@ +Copy or symlink this folder without renaming to your current working directory, and execute the bundle like so: +# timmy -c simplified-performance-testing/config.yaml diff --git a/timmy_data/simplified-performance-testing/scripts/cinder-VM-volume-write.sh b/timmy_data/simplified-performance-testing/scripts/cinder-VM-volume-write.sh new file mode 100644 index 0000000..ff41a2a --- /dev/null +++ b/timmy_data/simplified-performance-testing/scripts/cinder-VM-volume-write.sh @@ -0,0 +1,197 @@ +#!/bin/bash + +set -x + +# this test requires fully functional dd in VM, cirros (busybox) version will not work +# this test requires sudo and pkill in VM + +SPT_FLOATING_NET=${SPT_FLOATING_NET:-"admin_floating_net"} +SPT_FLAVOR=${SPT_FLAVOR:-"m1.small"} +SPT_IMAGE=${SPT_IMAGE:-"xenial"} +SPT_VM_USER=${SPT_VM_USER:-"ubuntu"} +SPT_VM_COOLDOWN=${SPT_VM_COOLDOWN:-"120"} +DD_OPTIONS=${DD_OPTIONS:-"oflag=direct"} +DD_TIMEOUT=${DD_TIMEOUT:-"10"} +SSH_OPTS="-q -n -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -l $SPT_VM_USER" +VM_BOOT_TIMEOUT_MINUTES=${VM_BOOT_TIMEOUT_MINUTES:-"5"} + +vm_boot_check_tries=$(($VM_BOOT_TIMEOUT_MINUTES*2)) +vm_boot_check_delay=30 +vm_active_check_tries=10 +vm_active_check_delay=10 +vm_volume_available_check_tries=10 +vm_volume_available_check_delay=30 +vm_volume_attached_check_tries=10 +vm_volume_attached_check_delay=10 + +function cleanup { + echo "cleaning up..." + [ -n "$instance_id" ] && [ -n "$floatingip" ] && nova floating-ip-disassociate $instance_id $floatingip + [ -n "$floatingip_id" ] && neutron floatingip-delete $floatingip_id + [ -n "$instance_id" ] && [ -n "$secgroup_id" ] && nova remove-secgroup $instance_id $secgroup_id + [ -n "$instance_id" ] && nova delete $instance_id + [ -n "$secgroup_id" ] && neutron security-group-delete $secgroup_id + [ -n "$router_id" ] && [ -n "$subnet_id" ] && neutron router-interface-delete $router_id $subnet_id + [ -n "$router_id" ] && neutron router-delete $router_id + [ -n "$subnet_id" ] && neutron subnet-delete $subnet_id + [ -n "$net_id" ] && neutron net-delete $net_id + [ -n "$keypair_name" ] && nova keypair-delete $keypair_name + [ -f "spt-temporary-keypair" ] && rm "spt-temporary-keypair" + [ -n "$volume_id" ] && cinder delete $volume_id +} + +function check_code { + # arguments: + # $1 - exit code + # $2 - error message + # $3 - command output to print + if [ "$1" -ne "0" ] + then + echo "$2:" + echo "$3" + cleanup + exit 1 + fi +} + +. openrc + +# create keypair +result="$(nova keypair-add "spt-temporary-keypair" >"spt-temporary-keypair" 2>&1)" +code=$? +[ "$code" -eq "0" ] && keypair_name="spt-temporary-keypair" +check_code $? "failed to create keypair" "$result" +chmod 600 "spt-temporary-keypair" + +# create network +result="$(neutron net-create "spt-temporary-net" 2>&1)" +check_code $? "failed to create network" "$result" +net_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# create subnet +result="$(neutron subnet-create --name "spt-temporary-subnet" $net_id 10.20.30.0/24 2>&1)" +check_code $? "failed to create subnet" "$result" +subnet_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# create router +result="$(neutron router-create "spt-temporary-router" 2>&1)" +check_code $? "failed to create router" "$result" +router_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# add router interface +result="$(neutron router-interface-add $router_id $subnet_id 2>&1)" +check_code $? "failed to add router interface to subnet" "$result" + +# set router gateway +result="$(neutron router-gateway-set $router_id $SPT_FLOATING_NET 2>&1)" +check_code $? "failed to set router gateway" "$result" + +# create security group +result="$(neutron security-group-create "spt-temporary-security-group" 2>&1)" +check_code $? "failed to create security group" "$result" +secgroup_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# create security group rule +result="$(neutron security-group-rule-create $secgroup_id --protocol TCP 2>&1)" +check_code $? "failed to create security group rule" "$result" + +# create floating ip +result="$(neutron floatingip-create $SPT_FLOATING_NET 2>&1)" +check_code $? "failed to create floatingip" "$result" +floatingip=$(echo "$result" | grep "^| floating_ip_address " | awk '{print $4}') +floatingip_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# boot VM +result="$(nova boot --image $SPT_IMAGE --flavor $SPT_FLAVOR --nic net-id=$net_id --key-name "spt-temporary-keypair" --security-groups $secgroup_id "spt-temporary-vm" 2>&1)" +check_code $? "failed to boot VM" "$result" +instance_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# wait for instance to become active +for i in $(seq 1 $vm_active_check_tries) +do + result="$(nova show $instance_id 2>&1)" + vm_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}') + [ "$vm_status" == "ACTIVE" ] && break + [ $i -lt $vm_active_check_tries ] && sleep $vm_active_check_delay +done +! [ "$vm_status" == "ACTIVE" ] && check_code 1 "timeout waiting for VM to become active" "$result" + +# associate floatingip +result="$(nova floating-ip-associate $instance_id $floatingip 2>&1)" +check_code $? "failed to associate floatingip" "$result" + +# create volume +result="$(cinder create --name "spt-temporary-volume" 4 2>&1)" +check_code $? "failed to create volume" "$result" +volume_id=$(echo "$result" | grep "^| *id " | awk '{print $4}') + +# wait for volume to become available +for i in $(seq 1 $vm_volume_available_check_tries) +do + result="$(cinder show $volume_id 2>&1)" + volume_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}') + [ "$volume_status" == "available" ] && break + [ $i -lt $vm_volume_available_check_tries ] && sleep $vm_volume_available_check_delay +done +! [ "$volume_status" == "available" ] && check_code 1 "timeout waiting for volume to become available" "$result" + +# test connection to VM +for i in $(seq 1 $vm_boot_check_tries) +do + result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "uptime" 2>&1)" + code=$? + [ $code -eq 0 ] && break + [ $i -lt $vm_boot_check_tries ] && sleep $vm_boot_check_delay +done +check_code $code "failed to connect to VM" "$result" + +# due to unreliable cinder attachment procedure, first get the list of current drives +vm_drives="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "find /dev/vd*" 2>&1)" +check_code $? "failed to find /dev/vd* in VM" "$result" + +# attach volume +result="$(nova volume-attach $instance_id $volume_id 2>&1)" +check_code $? "failed to attach volume" "$result" + +# wait for volume to appear in VM +for i in $(seq 1 $vm_volume_attached_check_tries) +do + vm_drives_2="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "find /dev/vd*" 2>&1)" + new_drive="$((echo "$vm_drives"; echo "$vm_drives_2") | sort | uniq -u)" + [ -n "$new_drive" ] && break + if [ $i -lt $vm_volume_attached_check_tries ] + then + nova volume-attach $instance_id $volume_id &> /dev/null # retry attaching since it is known to fail silently sometimes + sleep $vm_volume_attached_check_delay + fi +done +! [ -n "$new_drive" ] && check_code 1 "timeout waiting for volume to appear in VM" "$vm_drives_2" + +# mkfs in VM +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "sudo -i mkfs.ext4 -m 0 $new_drive" 2>&1)" +check_code $? "failed to create ext4 filesystem on VM" "$result" + +# mount volume in VM +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "sudo -i mount $new_drive /mnt && sudo -i chown $SPT_VM_USER /mnt" 2>&1)" +check_code $? "failed to mount ext4 filesystem on VM" "$result" + +# cooldown to allow VM to finish init activities +sleep $SPT_VM_COOLDOWN + +# give 10 seconds to dd to run, then getting intermediate stats, waiting 1 second and killing. Need to wait before killing, otherwise intermediate stats may not be printed +timelimit_cmd="pid=\$!; sleep $DD_TIMEOUT; kill -USR1 \$pid; sleep 1; kill \$pid" + +# run dd write tests +echo "running dd write test - writing 1GB with bs=4k" +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "dd if=/dev/zero of=/mnt/spt-write-test $DD_OPTIONS bs=4k count=262144 &:; $timelimit_cmd" 2>&1)" +echo "$result" | xargs -I@ echo "DD_TEST_1: @" + +echo "running dd write test - writing 1GB with bs=1M" +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "dd if=/dev/zero of=/mnt/spt-write-test-2 $DD_OPTIONS bs=1M count=1024 &:; $timelimit_cmd" 2>&1)" +echo "$result" | xargs -I@ echo "DD_TEST_2: @" + +echo "running dd write test - writing 1GB with bs=1G" +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "dd if=/dev/zero of=/mnt/spt-write-test-3 $DD_OPTIONS bs=1G count=1" 2>&1)" # no time limit here because it's a single block operation, no intermediate results available +echo "$result" | xargs -I@ echo "DD_TEST_3: @" + +cleanup diff --git a/timmy_data/simplified-performance-testing/scripts/glance-1-image-create.sh b/timmy_data/simplified-performance-testing/scripts/glance-1-image-create.sh new file mode 100644 index 0000000..28c2ff6 --- /dev/null +++ b/timmy_data/simplified-performance-testing/scripts/glance-1-image-create.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +. openrc +log='glance-image-upload.log' +rm $log +result="$(dd if=/dev/zero bs=1M count=4000 2>>$log | /usr/bin/time -f%e glance image-create --name "spt-test-image" --container-format bare --disk-format raw 2>>$log)" +cat $log +echo "$result" +rm $log diff --git a/timmy_data/simplified-performance-testing/scripts/glance-1-image-download.sh b/timmy_data/simplified-performance-testing/scripts/glance-1-image-download.sh new file mode 100644 index 0000000..0dd7bf5 --- /dev/null +++ b/timmy_data/simplified-performance-testing/scripts/glance-1-image-download.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -x + +. openrc +id=$(glance image-list | grep "spt-test-image" | cut -d' ' -f2) +/usr/bin/time -f%e glance image-download $id 2>&1 > /dev/null +glance image-delete $id diff --git a/timmy_data/simplified-performance-testing/scripts/iperf-client.sh b/timmy_data/simplified-performance-testing/scripts/iperf-client.sh new file mode 100644 index 0000000..bf7f159 --- /dev/null +++ b/timmy_data/simplified-performance-testing/scripts/iperf-client.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -x + +function timeout_kill() { + if [ "$1" -gt 0 ] + then + sleep 10.5 + [ "$(pgrep iperf | grep -c "$1")" -gt 0 ] && sleep 5 + [ "$(pgrep iperf | grep -c "$1")" -gt 0 ] && kill -9 $1 &>/dev/null + fi +} + +SPT_IPERF_PORT=${SPT_IPERF_PORT:-"65432"} +[ -z "$SERVER_IP" ] && echo '$SERVER_IP not provided, exiting' && exit 1 + +# install iperf +which iperf &>/dev/null +if [ "$?" -ne "0" ] +then + result="$(DEBIAN_FRONTEND=noninteractive apt-get -y install iperf 2>&1)" + [ "$?" -ne "0" ] && echo -e "failed to install iperf:\n$result" && exit 1 +fi + +iperf -c $SERVER_IP -p $SPT_IPERF_PORT & +timeout_kill $! +iperf -c $SERVER_IP -p $SPT_IPERF_PORT -P10 & +timeout_kill $! diff --git a/timmy_data/simplified-performance-testing/scripts/iperf-server-start.sh b/timmy_data/simplified-performance-testing/scripts/iperf-server-start.sh new file mode 100644 index 0000000..6c70130 --- /dev/null +++ b/timmy_data/simplified-performance-testing/scripts/iperf-server-start.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +SPT_IPERF_PORT=${SPT_IPERF_PORT:-"65432"} + +# install iperf +which iperf &>/dev/null +if [ "$?" -ne "0" ] +then + result="$(DEBIAN_FRONTEND=noninteractive apt-get -y install iperf 2>&1)" + [ "$?" -ne "0" ] && echo -e "failed to install iperf:\n$result" && exit 1 +fi + +# add firewall rule +if [ "$(iptables -S | grep -c spt-temporary-rule-tcp-${SPT_IPERF_PORT})" -eq 0 ] +then + result="$(iptables -I INPUT 1 -p tcp --dport ${SPT_IPERF_PORT} -j ACCEPT -m comment --comment "spt-temporary-rule-tcp-${SPT_IPERF_PORT}" 2>&1)" + [ "$?" -ne "0" ] && echo -e "failed to add iptables rule:\n$result" && exit 1 +fi + +# start iperf server +outfile=$(mktemp) +iperf -s -p $SPT_IPERF_PORT &> $outfile & +printf $outfile diff --git a/timmy_data/simplified-performance-testing/scripts/iperf-server-stop.sh b/timmy_data/simplified-performance-testing/scripts/iperf-server-stop.sh new file mode 100644 index 0000000..b03d0dc --- /dev/null +++ b/timmy_data/simplified-performance-testing/scripts/iperf-server-stop.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +SPT_IPERF_PORT=${SPT_IPERF_PORT:-"65432"} + +killall iperf +while ["$(iptables -L --line-numbers | grep -c 'spt-temporary-rule')" -ge 0 ] +do + rulenum=`iptables -L --line-numbers | grep 'spt-temporary-rule' | head -n 1 | awk '{print $1}'` + [ -n "$sulenum" ] && [ "$rulenum" -ge 0 ] && iptables -D INPUT $rulenum +done +if [ -n $SERVER_OUTPUT ] +then + echo $SERVER_OUTPUT + cat $SERVER_OUTPUT + rm -f $SERVER_OUTPUT +else + echo '$SERVER_OUTPUT not provided' +fi diff --git a/timmy_data/simplified-performance-testing/scripts/network-VM-to-VM-iperf-tests.sh b/timmy_data/simplified-performance-testing/scripts/network-VM-to-VM-iperf-tests.sh new file mode 100644 index 0000000..6a25d0e --- /dev/null +++ b/timmy_data/simplified-performance-testing/scripts/network-VM-to-VM-iperf-tests.sh @@ -0,0 +1,350 @@ +#!/bin/bash + +set -x + +# this test requires sudo in VM +# this test requires iperf in VM (or available for installation) +# this test expects iptables in VM to accept connections, it does not manipulate iptables inside VM + +SPT_DNS=${SPT_DNS:-"8.8.8.8"} +SPT_FLOATING_NET=${SPT_FLOATING_NET:-"admin_floating_net"} +SPT_FLAVOR=${SPT_FLAVOR:-"m1.small"} +SPT_IMAGE=${SPT_IMAGE:-"xenial"} +SPT_VM_USER=${SPT_VM_USER:-"ubuntu"} +SPT_VM_COOLDOWN=${SPT_VM_COOLDOWN:-"120"} +SPT_VM_INSTALL_COMMAND=${SPT_VM_INSTALL_COMMAND:-"apt-get"} +SPT_IPERF_PORT=${SPT_IPERF_PORT:-"65432"} +SPT_AVAILABILITY_ZONE=${SPT_AVAILABILITY_ZONE:-"nova"} +SSH_OPTS="-q -n -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -l $SPT_VM_USER" +VM_BOOT_TIMEOUT_MINUTES=${VM_BOOT_TIMEOUT_MINUTES:-"5"} + +vm_boot_check_tries=$(($VM_BOOT_TIMEOUT_MINUTES*2)) +vm_boot_check_delay=30 +vm_active_check_tries=10 +vm_active_check_delay=10 + +function partial_cleanup_2 { + [ -n "$instance_2_id" ] && [ -n "$floatingip_2" ] && nova floating-ip-disassociate $instance_2_id $floatingip_2 + [ -n "$instance_2_id" ] && [ -n "$secgroup_id" ] && nova remove-secgroup $instance_2_id $secgroup_id + [ -n "$instance_2_id" ] && nova delete $instance_2_id +} + +function cleanup { + echo "cleaning up..." + partial_cleanup_2 + [ -n "$floatingip_2_id" ] && neutron floatingip-delete $floatingip_2_id + [ -n "$instance_id" ] && [ -n "$floatingip" ] && nova floating-ip-disassociate $instance_id $floatingip + [ -n "$floatingip_id" ] && neutron floatingip-delete $floatingip_id + [ -n "$instance_id" ] && [ -n "$secgroup_id" ] && nova remove-secgroup $instance_id $secgroup_id + [ -n "$instance_id" ] && nova delete $instance_id + [ -n "$secgroup_id" ] && neutron security-group-delete $secgroup_id + [ -n "$router_id" ] && [ -n "$subnet_id" ] && neutron router-interface-delete $router_id $subnet_id + [ -n "$router_id" ] && [ -n "$subnet_2_id" ] && neutron router-interface-delete $router_id $subnet_2_id + [ -n "$router_id" ] && neutron router-delete $router_id + [ -n "$subnet_id" ] && neutron subnet-delete $subnet_id + [ -n "$subnet_2_id" ] && neutron subnet-delete $subnet_2_id + [ -n "$net_id" ] && neutron net-delete $net_id + [ -n "$net_2_id" ] && neutron net-delete $net_2_id + [ -n "$keypair_name" ] && nova keypair-delete $keypair_name + [ -f "spt-temporary-keypair" ] && rm "spt-temporary-keypair" +} + +function check_code { + # arguments: + # $1 - exit code + # $2 - error message + # $3 - command output to print + if [ "$1" -ne "0" ] + then + echo "$2:" + echo "$3" + cleanup + exit 1 + fi +} + + +. openrc + +# create keypair +result="$(nova keypair-add "spt-temporary-keypair" >"spt-temporary-keypair" 2>&1)" +code=$? +[ "$code" -eq "0" ] && keypair_name="spt-temporary-keypair" +check_code $? "failed to create keypair" "$result" +chmod 600 "spt-temporary-keypair" + +# create network +result="$(neutron net-create "spt-temporary-net" 2>&1)" +check_code $? "failed to create network" "$result" +net_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# create subnet +result="$(neutron subnet-create --name "spt-temporary-subnet" --dns-nameserver $SPT_DNS $net_id 10.20.30.0/24 2>&1)" +check_code $? "failed to create subnet" "$result" +subnet_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# create router +result="$(neutron router-create "spt-temporary-router" 2>&1)" +check_code $? "failed to create router" "$result" +router_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# add router interface +result="$(neutron router-interface-add $router_id $subnet_id 2>&1)" +check_code $? "failed to add router interface to subnet" "$result" + +# set router gateway +result="$(neutron router-gateway-set $router_id $SPT_FLOATING_NET 2>&1)" +check_code $? "failed to set router gateway" "$result" + +# create security group +result="$(neutron security-group-create "spt-temporary-security-group" 2>&1)" +check_code $? "failed to create security group" "$result" +secgroup_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# create security group rule +result="$(neutron security-group-rule-create $secgroup_id --protocol TCP 2>&1)" +check_code $? "failed to create security group rule" "$result" + +# create floating ip +result="$(neutron floatingip-create $SPT_FLOATING_NET 2>&1)" +check_code $? "failed to create floatingip" "$result" +floatingip=$(echo "$result" | grep "^| floating_ip_address " | awk '{print $4}') +floatingip_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# boot VM +result="$(nova boot --image $SPT_IMAGE --flavor $SPT_FLAVOR --nic net-id=$net_id --key-name "spt-temporary-keypair" --security-groups $secgroup_id --availability-zone $SPT_AVAILABILITY_ZONE "spt-temporary-vm" 2>&1)" +check_code $? "failed to boot VM" "$result" +instance_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# wait for instance to become active +for i in $(seq 1 $vm_active_check_tries) +do + result="$(nova show $instance_id 2>&1)" + vm_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}') + [ "$vm_status" == "ACTIVE" ] && break + [ $i -lt $vm_active_check_tries ] && sleep $vm_active_check_delay +done +! [ "$vm_status" == "ACTIVE" ] && check_code 1 "timeout waiting for VM to become active" "$result" + +# associate floatingip +result="$(nova floating-ip-associate $instance_id $floatingip 2>&1)" +check_code $? "failed to associate floatingip" "$result" + +# get fixed ip +result="$(neutron floatingip-show $floatingip_id 2>&1)" +check_code $? "failed to show floatingip info" "$result" +fixedip=$(echo "$result" | grep "^| fixed_ip_address " | awk '{print $4}') + +# test connection to VM +for i in $(seq 1 $vm_boot_check_tries) +do + result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "uptime" 2>&1)" + code=$? + [ $code -eq 0 ] && break + [ $i -lt $vm_boot_check_tries ] && sleep $vm_boot_check_delay +done +check_code $code "failed to connect to VM" "$result" + +# install iperf +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip "sudo $SPT_VM_INSTALL_COMMAND install iperf" 2>&1)" +check_code $? "failed to install iperf" "$result" + +# start server +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip ' +outfile=$(mktemp) +iperf -s -p '$SPT_IPERF_PORT'&> $outfile & +pid=$! +if ! [ $(pgrep iperf) ] +then + echo "iperf did not start" + cat $outfile + rm -f $outfile + exit 1 +fi +echo $pid +echo $outfile' 2>&1)" +check_code $? "failed to start iperf server" "$result" +read -d'\n' server_pid server_output <<< "$result" + +########################################################## +### stage 1 - test with another VM on the same compute ### +########################################################## + +# find location of the first VM + +host=$(mysql -s -N -D nova -e "select host from instances where uuid='$instance_id'" 2>&1) +check_code $? "failed to get host of VM $instance_id" "$host" + +# allocate second floating IP +result="$(neutron floatingip-create $SPT_FLOATING_NET 2>&1)" +check_code $? "failed to create second floatingip" "$result" +floatingip_2=$(echo "$result" | grep "^| floating_ip_address " | awk '{print $4}') +floatingip_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# boot second VM on the same host +result="$(nova boot --image $SPT_IMAGE --flavor $SPT_FLAVOR --nic net-id=$net_id --key-name "spt-temporary-keypair" --security-groups $secgroup_id --availability-zone $SPT_AVAILABILITY_ZONE:$host "spt-temporary-vm-2" 2>&1)" +check_code $? "failed to boot second VM on the same host" "$result" +instance_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# wait for instance to become active +for i in $(seq 1 $vm_active_check_tries) +do + result="$(nova show $instance_2_id 2>&1)" + vm_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}') + [ "$vm_status" == "ACTIVE" ] && break + [ $i -lt $vm_active_check_tries ] && sleep $vm_active_check_delay +done +! [ "$vm_status" == "ACTIVE" ] && check_code 1 "timeout waiting for second VM to become active" "$result" + +# associate floatingip +result="$(nova floating-ip-associate $instance_2_id $floatingip_2 2>&1)" +check_code $? "failed to associate floatingip_2" "$result" + +# test connection to VM +for i in $(seq 1 $vm_boot_check_tries) +do + result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "uptime" 2>&1)" + code=$? + [ $code -eq 0 ] && break + [ $i -lt $vm_boot_check_tries ] && sleep $vm_boot_check_delay +done +check_code $code "failed to connect to second VM" "$result" + +# install iperf on second VM +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "sudo $SPT_VM_INSTALL_COMMAND install iperf" 2>&1)" +check_code $? "failed to install iperf on second VM" "$result" + +# run VM to VM test via internal IPs +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "iperf -c $fixedip -p $SPT_IPERF_PORT" 2>&1)" +echo "VM to VM, same host, internal IP, 1 connection:" +echo "$result" + +# removing second VM +partial_cleanup_2 + +##################################################### +### stage 2 - test with VMs on different computes ### +##################################################### + +#find current AZ id +az_id=$(mysql -s -N -D nova -e "select aggregate_id from aggregate_hosts where host='$host' and deleted = 0" 2>&1) + +# find another host +if [ -z "$az_id" ] +then + # no AZ defined - all hosts in the default "nova" AZ + host_2=$(mysql -s -N -D nova -e "select host from compute_nodes where host <> '$host' and deleted = 0 limit 1" 2>&1) +else + host_2=$(mysql -s -N -D nova -e "select host from aggregate_hosts where aggregate_id = $az_id and host <> '$host' and deleted = 0 limit 1" 2>&1) +fi +check_code $? "failed to get a different host for a new VM" "$host" + +# boot second VM on a different host +result="$(nova boot --image $SPT_IMAGE --flavor $SPT_FLAVOR --nic net-id=$net_id --key-name "spt-temporary-keypair" --security-groups $secgroup_id --availability-zone $SPT_AVAILABILITY_ZONE:$host_2 "spt-temporary-vm-2" 2>&1)" +check_code $? "failed to boot second VM on a different host" "$result" +instance_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# wait for instance to become active +for i in $(seq 1 $vm_active_check_tries) +do + result="$(nova show $instance_2_id 2>&1)" + vm_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}') + [ "$vm_status" == "ACTIVE" ] && break + [ $i -lt $vm_active_check_tries ] && sleep $vm_active_check_delay +done +! [ "$vm_status" == "ACTIVE" ] && check_code 1 "timeout waiting for second VM to become active" "$result" + +# associate floatingip +result="$(nova floating-ip-associate $instance_2_id $floatingip_2 2>&1)" +check_code $? "failed to associate floatingip_2" "$result" + +# test connection to VM +for i in $(seq 1 $vm_boot_check_tries) +do + result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "uptime" 2>&1)" + code=$? + [ $code -eq 0 ] && break + [ $i -lt $vm_boot_check_tries ] && sleep $vm_boot_check_delay +done +check_code $code "failed to connect to second VM" "$result" + +# install iperf on second VM +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "sudo $SPT_VM_INSTALL_COMMAND install iperf" 2>&1)" +check_code $? "failed to install iperf on second VM" "$result" + +# run VM to VM test via internal IPs +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "iperf -c $fixedip -p $SPT_IPERF_PORT" 2>&1)" +echo "VM to VM, different hosts, internal IP, 1 connection:" +echo "$result" + +# run VM to VM test via internal IPs, 10 connections +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "iperf -c $fixedip -p $SPT_IPERF_PORT -P10" 2>&1)" +echo "VM to VM, different hosts, internal IP, 10 connections:" +echo "$result" + +# run VM to VM test via floating IPs +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "iperf -c $floatingip -p $SPT_IPERF_PORT" 2>&1)" +echo "VM to VM, different hosts, floating IP, 1 connection:" +echo "$result" + +# removing second VM +partial_cleanup_2 + +##################################################################################################################### +### stage 3 - test with VMs on different hosts and connected to different networks, networks connected via router ### +##################################################################################################################### + +# create second network +result="$(neutron net-create "spt-temporary-net-2" 2>&1)" +check_code $? "failed to create second network" "$result" +net_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# create second subnet +result="$(neutron subnet-create --name "spt-temporary-subnet-2" --dns-nameserver $SPT_DNS $net_2_id 10.20.40.0/24 2>&1)" +check_code $? "failed to create second subnet" "$result" +subnet_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# join subnets via router +result="$(neutron router-interface-add $router_id $subnet_2_id 2>&1)" +check_code $? "failed to add router interface to second subnet" "$result" + +# boot second VM in second network +result="$(nova boot --image $SPT_IMAGE --flavor $SPT_FLAVOR --nic net-id=$net_2_id --key-name "spt-temporary-keypair" --security-groups $secgroup_id --availability-zone $SPT_AVAILABILITY_ZONE:$host_2 "spt-temporary-vm-2" 2>&1)" +check_code $? "failed to boot second VM" "$result" +instance_2_id=$(echo "$result" | grep "^| id " | awk '{print $4}') + +# wait for instance to become active +for i in $(seq 1 $vm_active_check_tries) +do + result="$(nova show $instance_2_id 2>&1)" + vm_status=$(echo "$result" | grep "^| *status" | awk '{printf $4}') + [ "$vm_status" == "ACTIVE" ] && break + [ $i -lt $vm_active_check_tries ] && sleep $vm_active_check_delay +done +! [ "$vm_status" == "ACTIVE" ] && check_code 1 "timeout waiting for second VM to become active" "$result" + +# associate floatingip +result="$(nova floating-ip-associate $instance_2_id $floatingip_2 2>&1)" +check_code $? "failed to associate floatingip_2" "$result" + +# test connection to VM +for i in $(seq 1 $vm_boot_check_tries) +do + result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "uptime" 2>&1)" + code=$? + [ $code -eq 0 ] && break + [ $i -lt $vm_boot_check_tries ] && sleep $vm_boot_check_delay +done +check_code $code "failed to connect to second VM" "$result" + +# install iperf on second VM +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "sudo $SPT_VM_INSTALL_COMMAND install iperf" 2>&1)" +check_code $? "failed to install iperf on second VM" "$result" + +# run VM to VM test via internal IPs +result="$(ssh $SSH_OPTS -i spt-temporary-keypair $floatingip_2 "iperf -c $fixedip -p $SPT_IPERF_PORT" 2>&1)" +echo "VM to VM, different hosts, different networks, internal IP, 1 connection:" +echo "$result" + +cleanup +