Add zuul job with integration testing

This patch introduces base integration test that verifies
messaging loop between agent and server, command execution
and report generation. Shaker in tested in standalone
mode, without intergration with OpenStack.

Change-Id: I8fdc63abdf0be0fc67c3b438bbfc1cf2f69c9f15

.zuul.yaml

@@ -10,7 +10,32 @@
     check:
       jobs:
         - openstack-tox-docs
+        - shaker-integration-py27
+        - shaker-integration-py35
     gate:
       jobs:
         - openstack-tox-docs
+        - shaker-integration-py27
+        - shaker-integration-py35
 
+- job:
+    name: shaker-integration-py27
+    parent: openstack-tox
+    voting: false
+    description: |
+      Run integration tests under Python 2.7
+
+      To run tests manually use ``tox -e integration-py27`` command.
+    vars:
+      tox_envlist: integration-py27
+
+- job:
+    name: shaker-integration-py35
+    parent: openstack-tox
+    voting: false
+    description: |
+      Run integration tests under Python 3.5
+
+      To run tests manually use ``tox -e integration-py35`` command.
+    vars:
+      tox_envlist: integration-py35

shaker/scenarios/test/spot.yaml

@@ -0,0 +1,14 @@
+title: Static agents
+
+description:
+  In this scenario Shaker runs tests in spot mode.
+  The scenario can be used for Shaker integration testing.
+
+execution:
+  tests:
+  -
+    title: Verify spot mode
+    class: shell
+    program: echo "Hello, world!"
+    sla:
+    - "[type == 'agent'] >> (stdout & '.*Hello.*')"

shaker/scenarios/test/static_agent.yaml

@@ -13,16 +13,16 @@ deployment:
 execution:
   tests:
   -
-    title: List all files
+    title: Verify shell executor
     class: shell
-    program: ls -al
+    program: echo "Hello, world!"
     sla:
     - "[type == 'agent'] >> (stderr == '')"
   -
-    title: Run sample script
+    title: Verify script executor
     class: shell
     script: |
       #!/bin/bash
-      echo "hello world"
+      echo "Let us print hello world remotely"
     sla:
     - "[type == 'agent'] >> (stdout & '.*hello world.*')"

tools/process_subunit.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+LOGDIR=$1
+
+SUBUNIT_FILE="${LOGDIR}/job.subunit"
+HTML_FILE="${LOGDIR}/job.html"
+
+ls ${LOGDIR}/*.subunit | xargs cat > ${SUBUNIT_FILE}
+
+subunit2html ${SUBUNIT_FILE} ${HTML_FILE}
+
+cat ${SUBUNIT_FILE} | subunit-stats

tools/run_agent_as_daemon.py

@@ -0,0 +1,74 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a helper script for integration testing that runs agent as daemon
+
+import argparse
+import os
+import signal
+import sys
+
+from daemonize import Daemonize
+
+from shaker.agent import agent
+
+
+def stop_daemon(pid_file_name):
+    try:
+        with open(pid_file_name) as fd:
+            pid = fd.read()
+    except IOError:
+        return
+
+    try:
+        os.kill(int(pid), signal.SIGKILL)
+    except Exception:
+        pass
+
+    try:
+        os.remove(pid_file_name)
+    except Exception:
+        pass
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Shaker agent')
+    parser.add_argument('--start',
+                        action='store_true',
+                        help='Start the daemon')
+    parser.add_argument('--stop',
+                        action='store_true',
+                        help='Stop the daemon')
+    parser.add_argument('--pid',
+                        help='Name of file where PID is stored')
+    args = parser.parse_args()
+
+    def start():
+        sys.argv = sys.argv[:1]
+        agent.main()
+
+    pid_file_name = args.pid
+
+    if args.start:
+        if os.path.exists(pid_file_name):
+            stop_daemon(pid_file_name)
+        daemon = Daemonize(app="shaker-agent", pid=pid_file_name, action=start)
+        daemon.start()
+    elif args.stop:
+        stop_daemon(pid_file_name)
+    else:
+        start()
+
+
+if __name__ == '__main__':
+    main()

tox.ini

@@ -34,6 +34,37 @@ commands =
     coverage html -d cover
     coverage xml -o cover/coverage.xml
 
+[testenv:integration]
+setenv = {[testenv]setenv}
+         SHAKER_SERVER_ENDPOINT=localhost:5900
+         SHAKER_AGENT_ID=the-agent
+         SHAKER_POLLING_INTERVAL=2
+         SHAKER_AGENT_JOIN_TIMEOUT=30
+deps = {[testenv]deps}
+       daemonize
+       os-testr
+whitelist_externals = bash
+commands =
+    shaker --debug --scenario test/spot --artifacts-dir {envlogdir}
+    python tools/run_agent_as_daemon.py --start --pid {envtmpdir}/shaker-agent.pid
+    shaker --debug --scenario test/static_agent --artifacts-dir {envlogdir}
+    python tools/run_agent_as_daemon.py --stop --pid {envtmpdir}/shaker-agent.pid
+    bash tools/process_subunit.sh {envlogdir}
+
+[testenv:integration-py27]
+basepython = python2.7
+setenv = {[testenv:integration]setenv}
+deps = {[testenv:integration]deps}
+whitelist_externals = {[testenv:integration]whitelist_externals}
+commands = {[testenv:integration]commands}
+
+[testenv:integration-py35]
+basepython = python3.5
+setenv = {[testenv:integration]setenv}
+deps = {[testenv:integration]deps}
+whitelist_externals = {[testenv:integration]whitelist_externals}
+commands = {[testenv:integration]commands}
+
 [testenv:genconfig]
 # When shaker is setup in develop mode it results in 2 packages: shaker and pyshaker
 # The workaround is to setup it in production mode