From 1119ec943facbf9e5681521158e8216c19a3d7e8 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Fri, 19 Oct 2018 12:12:53 +0200 Subject: [PATCH] Add zuul job with integration testing This patch introduces base integration test that verifies messaging loop between agent and server, command execution and report generation. Shaker in tested in standalone mode, without intergration with OpenStack. Change-Id: I8fdc63abdf0be0fc67c3b438bbfc1cf2f69c9f15 --- .zuul.yaml | 25 +++++++ shaker/scenarios/test/spot.yaml | 14 ++++ .../{misc => test}/static_agent.yaml | 8 +- .../{misc => test}/static_agents_pair.yaml | 0 tools/process_subunit.sh | 12 +++ tools/run_agent_as_daemon.py | 74 +++++++++++++++++++ tox.ini | 31 ++++++++ 7 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 shaker/scenarios/test/spot.yaml rename shaker/scenarios/{misc => test}/static_agent.yaml (74%) rename shaker/scenarios/{misc => test}/static_agents_pair.yaml (100%) create mode 100755 tools/process_subunit.sh create mode 100644 tools/run_agent_as_daemon.py diff --git a/.zuul.yaml b/.zuul.yaml index 41b8d4f..48405df 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -10,7 +10,32 @@ check: jobs: - openstack-tox-docs + - shaker-integration-py27 + - shaker-integration-py35 gate: jobs: - openstack-tox-docs + - shaker-integration-py27 + - shaker-integration-py35 +- job: + name: shaker-integration-py27 + parent: openstack-tox + voting: false + description: | + Run integration tests under Python 2.7 + + To run tests manually use ``tox -e integration-py27`` command. + vars: + tox_envlist: integration-py27 + +- job: + name: shaker-integration-py35 + parent: openstack-tox + voting: false + description: | + Run integration tests under Python 3.5 + + To run tests manually use ``tox -e integration-py35`` command. + vars: + tox_envlist: integration-py35 diff --git a/shaker/scenarios/test/spot.yaml b/shaker/scenarios/test/spot.yaml new file mode 100644 index 0000000..d27d5a9 --- /dev/null +++ b/shaker/scenarios/test/spot.yaml @@ -0,0 +1,14 @@ +title: Static agents + +description: + In this scenario Shaker runs tests in spot mode. + The scenario can be used for Shaker integration testing. + +execution: + tests: + - + title: Verify spot mode + class: shell + program: echo "Hello, world!" + sla: + - "[type == 'agent'] >> (stdout & '.*Hello.*')" diff --git a/shaker/scenarios/misc/static_agent.yaml b/shaker/scenarios/test/static_agent.yaml similarity index 74% rename from shaker/scenarios/misc/static_agent.yaml rename to shaker/scenarios/test/static_agent.yaml index f577da1..bc07c90 100644 --- a/shaker/scenarios/misc/static_agent.yaml +++ b/shaker/scenarios/test/static_agent.yaml @@ -13,16 +13,16 @@ deployment: execution: tests: - - title: List all files + title: Verify shell executor class: shell - program: ls -al + program: echo "Hello, world!" sla: - "[type == 'agent'] >> (stderr == '')" - - title: Run sample script + title: Verify script executor class: shell script: | #!/bin/bash - echo "hello world" + echo "Let us print hello world remotely" sla: - "[type == 'agent'] >> (stdout & '.*hello world.*')" diff --git a/shaker/scenarios/misc/static_agents_pair.yaml b/shaker/scenarios/test/static_agents_pair.yaml similarity index 100% rename from shaker/scenarios/misc/static_agents_pair.yaml rename to shaker/scenarios/test/static_agents_pair.yaml diff --git a/tools/process_subunit.sh b/tools/process_subunit.sh new file mode 100755 index 0000000..48aba79 --- /dev/null +++ b/tools/process_subunit.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +LOGDIR=$1 + +SUBUNIT_FILE="${LOGDIR}/job.subunit" +HTML_FILE="${LOGDIR}/job.html" + +ls ${LOGDIR}/*.subunit | xargs cat > ${SUBUNIT_FILE} + +subunit2html ${SUBUNIT_FILE} ${HTML_FILE} + +cat ${SUBUNIT_FILE} | subunit-stats diff --git a/tools/run_agent_as_daemon.py b/tools/run_agent_as_daemon.py new file mode 100644 index 0000000..f468d8c --- /dev/null +++ b/tools/run_agent_as_daemon.py @@ -0,0 +1,74 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a helper script for integration testing that runs agent as daemon + +import argparse +import os +import signal +import sys + +from daemonize import Daemonize + +from shaker.agent import agent + + +def stop_daemon(pid_file_name): + try: + with open(pid_file_name) as fd: + pid = fd.read() + except IOError: + return + + try: + os.kill(int(pid), signal.SIGKILL) + except Exception: + pass + + try: + os.remove(pid_file_name) + except Exception: + pass + + +def main(): + parser = argparse.ArgumentParser(description='Shaker agent') + parser.add_argument('--start', + action='store_true', + help='Start the daemon') + parser.add_argument('--stop', + action='store_true', + help='Stop the daemon') + parser.add_argument('--pid', + help='Name of file where PID is stored') + args = parser.parse_args() + + def start(): + sys.argv = sys.argv[:1] + agent.main() + + pid_file_name = args.pid + + if args.start: + if os.path.exists(pid_file_name): + stop_daemon(pid_file_name) + daemon = Daemonize(app="shaker-agent", pid=pid_file_name, action=start) + daemon.start() + elif args.stop: + stop_daemon(pid_file_name) + else: + start() + + +if __name__ == '__main__': + main() diff --git a/tox.ini b/tox.ini index 88097c0..83cccf0 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,37 @@ commands = coverage html -d cover coverage xml -o cover/coverage.xml +[testenv:integration] +setenv = {[testenv]setenv} + SHAKER_SERVER_ENDPOINT=localhost:5900 + SHAKER_AGENT_ID=the-agent + SHAKER_POLLING_INTERVAL=2 + SHAKER_AGENT_JOIN_TIMEOUT=30 +deps = {[testenv]deps} + daemonize + os-testr +whitelist_externals = bash +commands = + shaker --debug --scenario test/spot --artifacts-dir {envlogdir} + python tools/run_agent_as_daemon.py --start --pid {envtmpdir}/shaker-agent.pid + shaker --debug --scenario test/static_agent --artifacts-dir {envlogdir} + python tools/run_agent_as_daemon.py --stop --pid {envtmpdir}/shaker-agent.pid + bash tools/process_subunit.sh {envlogdir} + +[testenv:integration-py27] +basepython = python2.7 +setenv = {[testenv:integration]setenv} +deps = {[testenv:integration]deps} +whitelist_externals = {[testenv:integration]whitelist_externals} +commands = {[testenv:integration]commands} + +[testenv:integration-py35] +basepython = python3.5 +setenv = {[testenv:integration]setenv} +deps = {[testenv:integration]deps} +whitelist_externals = {[testenv:integration]whitelist_externals} +commands = {[testenv:integration]commands} + [testenv:genconfig] # When shaker is setup in develop mode it results in 2 packages: shaker and pyshaker # The workaround is to setup it in production mode