Add test for full CiCd flow

Added new test that check deployment of CiCd app
and goes throught the full flow with deploying app to Tomcat.

Change-Id: I7ca7bd4182deb87615ba312a0c78f0fb37b91440

test-requirements.txt

@@ -6,3 +6,5 @@ python-muranoclient
 python-heatclient
 python-novaclient
 
+python-jenkins
+git-review

tests/base.py

@@ -16,11 +16,13 @@
 import json
 import logging
 import os
-import shutil
 import socket
+import shutil
 import time
 import uuid
+from xml.etree import ElementTree as et
 
+import jenkins
 import paramiko
 import requests
 import testtools
@@ -75,8 +77,8 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
 
         # Since its really useful to debug deployment after it fail lets
         # add such possibility
-        self.os_cleanup_before = str2bool('OS_CLEANUP_BEFORE', False)
+        self.os_cleanup_before = str2bool('OS_CLEANUP_BEFORE', True)
-        self.os_cleanup_after = str2bool('OS_CLEANUP_AFTER', True)
+        self.os_cleanup_after = str2bool('OS_CLEANUP_AFTER', False)
 
         # Data for Nodepool app
         self.os_np_username = os.environ.get('OS_NP_USERNAME', self.os_username)
@@ -100,8 +102,9 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
         # Application instance parameters
         self.flavor = os.environ.get('OS_FLAVOR', 'm1.medium')
         self.image = os.environ.get('OS_IMAGE')
+        self.docker_image = os.environ.get('OS_DOCKER_IMAGE')
         self.files = []
-        self.keyname, self.key_file = self._create_keypair()
+        self.keyname, self.pr_key, self.pub_key = self._create_keypair()
         self.availability_zone = os.environ.get('OS_ZONE', 'nova')
 
         self.envs = []
@@ -163,8 +166,14 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
         pr_key_file = self.create_file(
             'id_{}'.format(kp_name), keypair.private_key
         )
-        self.create_file('id_{}.pub'.format(kp_name), keypair.public_key)
+        # Note: by default, permissions of created file with
-        return kp_name, pr_key_file
+        # private keypair is too open
+        os.chmod(pr_key_file, 0600)
+
+        pub_key_file = self.create_file(
+            'id_{}.pub'.format(kp_name), keypair.public_key
+        )
+        return kp_name, pr_key_file, pub_key_file
 
     def _get_stack(self, environment_id):
         for stack in self.heat.stacks.list():
@@ -269,7 +278,7 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
             try:
                 ssh = paramiko.SSHClient()
                 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-                ssh.connect(fip, username='ubuntu', key_filename=self.key_file)
+                ssh.connect(fip, username='ubuntu', key_filename=self.pr_key)
                 ftp = ssh.open_sftp()
                 ftp.get(
                     '/var/log/murano-agent.log',
@@ -368,23 +377,18 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
                 self.fail('{} port is not opened on instance'.format(port))
 
     def check_url_access(self, ip, path, port):
-        attempt = 0
         proto = 'http' if port not in (443, 8443) else 'https'
-        url = '%s://%s:%s/%s' % (proto, ip, port, path)
+        url = '{proto}://{ip}:{port}/{path}'.format(
-
+            proto=proto,
-        while attempt < 5:
+            ip=ip,
-            resp = requests.get(url)
+            port=port,
-            if resp.status_code == 200:
+            path=path
-                LOG.debug('Service path "{}" is available'.format(url))
-                return
-            else:
-                time.sleep(5)
-                attempt += 1
-
-        self.fail(
-            'Service path {0} is unavailable after 5 attempts'.format(url)
         )
 
+        resp = requests.get(url, timeout=60)
+
+        return resp.status_code
+
     def deployment_success_check(self, environment, services_map):
         deployment = self.murano.deployments.list(environment.id)[-1]
 
@@ -414,3 +418,193 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
                     services_map[service]['url'],
                     services_map[service]['url_port']
                 )
+
+    @staticmethod
+    def add_to_file(path_to_file, context):
+        with open(path_to_file, "a") as f:
+            f.write(context)
+
+    @staticmethod
+    def read_from_file(path_to_file):
+        with open(path_to_file, "r") as f:
+            return f.read()
+
+    def execute_cmd_on_remote_host(self, host, cmd, key_file, user='ubuntu'):
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        client.connect(hostname=host, username=user, key_filename=key_file)
+        stdin, stdout, stderr = client.exec_command(cmd)
+        data = stdout.read() + stderr.read()
+        client.close()
+
+        return data
+
+    def export_ssh_options(self):
+        context = (
+            '#!/bin/bash\n'
+            'ssh -o StrictHostKeyChecking=no -i {0} "$@"'.format(self.pr_key)
+        )
+        gitwrap = self.create_file('/tmp/gitwrap.sh', context)
+        os.chmod(gitwrap, 0744)
+        os.environ['GIT_SSH'] = gitwrap
+
+    def clone_repo(self, gerrit_host, gerrit_user, repo, dest_dir):
+        repo_dest_path = os.path.join(dest_dir, repo.split('/')[-1])
+        self.files.append(repo_dest_path)
+
+        if os.path.isdir(repo_dest_path):
+            shutil.rmtree(repo_dest_path)
+
+        os.system('git clone ssh://{user}@{host}:29418/{repo} {dest}'.format(
+            user=gerrit_user,
+            host=gerrit_host,
+            repo=repo,
+            dest=repo_dest_path
+        ))
+        return repo_dest_path
+
+    def switch_to_branch(self, repo, branch):
+        os.system('cd {repo}; git checkout {branch}'.format(
+            repo=repo,
+            branch=branch)
+        )
+
+    def add_committer_info(self, configfile, user, email):
+        author_data = """
+        [user]
+            name={0}
+            email={1}
+        """.format(user, email)
+        self.add_to_file(configfile, author_data)
+
+    def make_commit(self, repo, branch, key, msg):
+        # NOTE need to think how to use GIT_SSH
+        os.system(
+            'cd {repo};'
+            'git add . ; git commit -am "{msg}"; '
+            'ssh-agent bash -c "ssh-add {key}; '
+            'git-review -r origin {branch}"'.format(
+                repo=repo,
+                msg=msg,
+                key=key,
+                branch=branch
+            )
+        )
+
+    @staticmethod
+    def _gerrit_cmd(gerrit_host, cmd):
+        return (
+            'sudo su -c "ssh -p 29418 -i '
+            '/home/gerrit2/review_site/etc/ssh_project_rsa_key '
+            'project-creator@{host} {cmd}" '
+            'gerrit2'.format(host=gerrit_host, cmd=cmd)
+        )
+
+    def get_last_open_patch(self, gerrit_ip, gerrit_host, project, commit_msg):
+        cmd = (
+            'gerrit query --format JSON status:open '
+            'project:{project} limit:1'.format(project=project)
+        )
+        cmd = self._gerrit_cmd(gerrit_host, cmd)
+
+        # Note: "gerrit query" returns results describing changes that
+        # match the input query.
+        # Here is an example of results for above query:
+        # {"project":"open-paas/project-config" ... "number":"1",
+        # {"type":"stats","rowCount":1,"runTimeMilliseconds":219, ...}
+        # Output has to be cut using "head -1", because json.loads can't
+        # decode multiple jsons
+
+        patch = self.execute_cmd_on_remote_host(
+            host=gerrit_ip,
+            key_file=self.pr_key,
+            cmd='{} | head -1'.format(cmd)
+        )
+        patch = json.loads(patch)
+        self.assertIn(commit_msg, patch['commitMessage'])
+
+        return patch['number']
+
+    def merge_commit(self, gerrit_ip, gerrit_host, project, commit_msg):
+        changeid = self.get_last_open_patch(
+            gerrit_ip=gerrit_ip,
+            gerrit_host=gerrit_host,
+            project=project,
+            commit_msg=commit_msg
+        )
+
+        cmd = (
+            'gerrit review --project {project} --verified +2 '
+            '--code-review +2 --label Workflow=+1 '
+            '--submit {id},1'.format(project=project, id=changeid)
+        )
+        cmd = self._gerrit_cmd(gerrit_host, cmd)
+
+        self.execute_cmd_on_remote_host(
+            host=gerrit_ip,
+            user='ubuntu',
+            key_file=self.pr_key,
+            cmd=cmd
+        )
+
+    def set_tomcat_ip(self, pom_file, ip):
+        et.register_namespace('', 'http://maven.apache.org/POM/4.0.0')
+        tree = et.parse(pom_file)
+        new_url = 'http://{ip}:8080/manager/text'.format(ip=ip)
+        ns = {'ns': 'http://maven.apache.org/POM/4.0.0'}
+        for plugin in tree.findall('ns:build/ns:plugins/', ns):
+            plugin_id = plugin.find('ns:artifactId', ns).text
+            if plugin_id == 'tomcat7-maven-plugin':
+                plugin.find('ns:configuration/ns:url', ns).text = new_url
+
+        tree.write(pom_file)
+
+    def get_gerrit_projects(self, gerrit_ip, gerrit_host):
+        cmd = self._gerrit_cmd(gerrit_host, 'gerrit ls-projects')
+        return self.execute_cmd_on_remote_host(
+            host=gerrit_ip,
+            user='ubuntu',
+            key_file=self.pr_key,
+            cmd=cmd
+        )
+
+    def get_jenkins_jobs(self, ip):
+        server = jenkins.Jenkins('http://{0}:8080'.format(ip))
+
+        return [job['name'] for job in server.get_all_jobs()]
+
+    def wait_for(self, func, expected, debug_msg, fail_msg, timeout, **kwargs):
+        LOG.debug(debug_msg)
+        start_time = time.time()
+
+        current = func(**kwargs)
+
+        def check(exp, cur):
+            if isinstance(cur, list) or isinstance(cur, str):
+                return exp not in cur
+            else:
+                return exp != cur
+
+        while check(expected, current):
+            current = func(**kwargs)
+
+            if time.time() - start_time > timeout:
+                self.fail("Time is out. {0}".format(fail_msg))
+            time.sleep(30)
+        LOG.debug('Expected result has been achieved.')
+
+    def get_last_build_number(self, ip, user, password, job_name, build_type):
+        server = jenkins.Jenkins(
+            'http://{0}:8080'.format(ip),
+            username=user,
+            password=password
+        )
+        # If there are no builds of desired type get_job_info returns None and
+        # it is not possible to get number, in this case this function returns
+        # None too and it means that there are no builds yet
+
+        build = server.get_job_info(job_name)[build_type]
+        if build:
+            return build['number']
+        else:
+            return build

tests/test_cicd_apps_flow.py

@@ -0,0 +1,333 @@
+# Copyright (c) 2016 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base
+
+
+class MuranoCiCdFlowTest(base.MuranoTestsBase):
+
+    def test_run_cicd_flow(self):
+        ldap_user = 'user'
+        ldap_password = 'P@ssw0rd'
+        ldap_user_email = 'email@example.com'
+
+        environment = self.create_env()
+        session = self.create_session(environment)
+
+        cicd_json = {
+            '?': {
+                '_{id}'.format(id=self.generate_id().hex): {'name': 'CI/CD'},
+                'id': str(self.generate_id()),
+                'type':
+                    'org.openstack.ci_cd_pipeline_murano_app.CiCdEnvironment'
+            },
+            'assignFloatingIp': True,
+            'availabilityZone': self.availability_zone,
+            'flavor': self.flavor,
+            'image': self.image,
+            'instance_name': environment.name,
+            'keyname': self.keyname,
+            'ldapEmail': ldap_user_email,
+            'ldapPass': 'P@ssw0rd',
+            'ldapRootEmail': 'root@example.com',
+            'ldapRootPass': ldap_password,
+            'ldapRootUser': 'root',
+            'ldapUser': ldap_user,
+            'userSSH': self.read_from_file(self.pub_key),
+            'name': 'CI/CD',
+        }
+
+        self.create_service(environment, session, cicd_json)
+        self.deploy_env(environment, session)
+
+        session = self.create_session(environment)
+        docker_json = {
+              'instance': {
+                  'name': self.rand_name('Docker'),
+                  'assignFloatingIp': True,
+                  'keyname': self.keyname,
+                  'flavor': 'm1.large',
+                  'image': self.docker_image,
+                  'availabilityZone': self.availability_zone,
+                  '?': {
+                      'type': 'io.murano.resources.LinuxMuranoInstance',
+                      'id': self.generate_id()
+                  },
+              },
+              'name': 'DockerVM',
+              '?': {
+                  '_{id}'.format(id=self.generate_id().hex): {
+                      'name': 'Docker VM Service'
+                  },
+                  'type': 'com.mirantis.docker.DockerStandaloneHost',
+                  'id': str(self.generate_id())
+              }
+        }
+
+        docker = self.create_service(environment, session, docker_json)
+
+        tomcat_json = {
+            'host': docker,
+            'image': 'tutum/tomcat',
+            'name': 'Tomcat',
+            'port': 8080,
+            'password': 'admin',
+            'publish': True,
+            '?': {
+                '_{id}'.format(id=self.generate_id().hex): {
+                    'name': 'Docker Tomcat'
+                },
+                'type': 'com.example.docker.DockerTomcat',
+                'id': str(self.generate_id())
+            }
+        }
+        self.create_service(environment, session, tomcat_json)
+
+        self.deploy_env(environment, session)
+
+        environment = self.get_env(environment)
+
+        check_services = {
+            'org.openstack.ci_cd_pipeline_murano_app.Jenkins': {
+                'ports': [8080, 22],
+                'url': 'api/',
+                'url_port': 8080
+            },
+            'org.openstack.ci_cd_pipeline_murano_app.Gerrit': {
+                'ports': [8081, 22],
+                'url': '#/admin/projects/',
+                'url_port': 8081
+            },
+            'org.openstack.ci_cd_pipeline_murano_app.OpenLDAP': {
+                'ports': [389, 22],
+                'url': None
+            },
+            'com.mirantis.docker.DockerStandaloneHost': {
+                'ports': [8080, 22],
+                'url': None
+            }
+        }
+
+        self.deployment_success_check(environment, check_services)
+
+        fips = self.get_services_fips(environment)
+
+        # Get Gerrit ip and hostname
+
+        gerrit_ip = fips['org.openstack.ci_cd_pipeline_murano_app.Gerrit']
+        gerrit_hostname = self.execute_cmd_on_remote_host(
+            host=gerrit_ip,
+            cmd='hostname -f',
+            key_file=self.pr_key
+        )[:-1]
+
+        self.export_ssh_options()
+
+        # Clone "project-config" repository
+
+        project_config_location = self.clone_repo(
+            gerrit_host=gerrit_ip,
+            gerrit_user=ldap_user,
+            repo='open-paas/project-config',
+            dest_dir='/tmp'
+        )
+
+        # Add new project to gerrit/projects.yaml
+
+        new_project = (
+            '- project: demo/petclinic\n'
+            '  description: petclinic new project\n'
+            '  upstream: https://github.com/sn00p/spring-petclinic\n'
+            '  acl-config: /home/gerrit2/acls/open-paas/project-config.config\n'
+        )
+        self.add_to_file(
+            '{0}/gerrit/projects.yaml'.format(project_config_location),
+            new_project
+        )
+
+        # Add committer info to project-config repo git config
+
+        self.add_committer_info(
+            configfile='{0}/.git/config'.format(project_config_location),
+            user=ldap_user,
+            email=ldap_user_email
+        )
+
+        # Make commit to project-config
+
+        self.make_commit(
+            repo=project_config_location,
+            branch='master',
+            key=self.pr_key,
+            msg='Add new project to gerrit/projects.yaml'
+        )
+
+        # Merge commit
+
+        self.merge_commit(
+            gerrit_ip=gerrit_ip,
+            gerrit_host=gerrit_hostname,
+            project='open-paas/project-config',
+            commit_msg='Add new project to gerrit/projects.yaml'
+        )
+
+        self.wait_for(
+            func=self.get_gerrit_projects,
+            expected='demo/petclinic',
+            debug_msg='Waiting while "demo/petlinic" project is created',
+            fail_msg='Project "demo/petclinic" wasn\'t created',
+            timeout=600,
+            gerrit_ip=gerrit_ip,
+            gerrit_host=gerrit_hostname,
+        )
+
+        # Create jenkins job for building petclinic app
+
+        new_job = (
+            '- project:\n'
+            '    name: petclinic\n'
+            '    jobs:\n'
+            '      - "{{name}}-java-app-deploy":\n'
+            '           git_url: "ssh://jenkins@{0}:29418/demo/petclinic"\n'
+            '           project: "demo/petclinic"\n'
+            '           branch: "Spring-Security"\n'
+            '           goals: tomcat7:deploy\n'.format(gerrit_hostname)
+        )
+        self.add_to_file(
+            '{0}/jenkins/jobs/projects.yaml'.format(project_config_location),
+            new_job
+        )
+
+        # Making commit to project-config
+
+        self.make_commit(
+            repo=project_config_location,
+            branch='master',
+            key=self.pr_key,
+            msg='Add job for petclinic app'
+        )
+
+        # Merge commit
+
+        self.merge_commit(
+            gerrit_ip=gerrit_ip,
+            gerrit_host=gerrit_hostname,
+            project='open-paas/project-config',
+            commit_msg='Add job for petclinic app'
+        )
+
+        # Wait while new "petclinic-java-app-deploy" job is created
+
+        self.wait_for(
+            func=self.get_jenkins_jobs,
+            expected='petclinic-java-app-deploy',
+            debug_msg='Waiting while "petclinic-java-app-deploy" is created',
+            fail_msg='Job "petclinic-java-app-deploy" wasn\'t created',
+            timeout=600,
+            ip=fips['org.openstack.ci_cd_pipeline_murano_app.Jenkins']
+        )
+
+        # Clone "demo/petclinic" repository
+
+        petclinic_location = self.clone_repo(
+            gerrit_host=gerrit_ip,
+            gerrit_user=ldap_user,
+            repo='demo/petclinic',
+            dest_dir='/tmp'
+        )
+
+        # Switch to "Spring-Security" branch
+
+        self.switch_to_branch(
+            repo=petclinic_location,
+            branch='Spring-Security'
+        )
+
+        # Set deployed Tomcat IP to pom.xml
+
+        self.set_tomcat_ip(
+            '{}/pom.xml'.format(petclinic_location),
+            fips['com.mirantis.docker.DockerStandaloneHost']
+        )
+
+        # Add committer info to demo/petclinic repo git config
+
+        self.add_committer_info(
+            configfile='{0}/.git/config'.format(petclinic_location),
+            user=ldap_user,
+            email=ldap_user_email
+        )
+
+        self.make_commit(
+            repo=petclinic_location,
+            branch='Spring-Security',
+            key=self.pr_key,
+            msg='Update Tomcat IP'
+        )
+
+        # Merge commit
+
+        self.merge_commit(
+            gerrit_ip=gerrit_ip,
+            gerrit_host=gerrit_hostname,
+            project='demo/petclinic',
+            commit_msg='Update Tomcat IP'
+        )
+
+        # Check that 'petclinic-java-app-deploy' (it triggers on-submit) was run
+
+        self.wait_for(
+            self.get_last_build_number,
+            expected=1,
+            debug_msg='Waiting while "petclinic-java-app-deploy" '
+                      'job is run and first build is completed',
+            fail_msg='Job "petclinic-java-app-deploy" wasn\'t run on-submit',
+            timeout=900,
+            ip=fips['org.openstack.ci_cd_pipeline_murano_app.Jenkins'],
+            user=ldap_user,
+            password=ldap_password,
+            job_name='petclinic-java-app-deploy',
+            build_type='lastCompletedBuild'
+        )
+
+        # Check that 'petclinic-java-app-deploy' (it triggers on-submit) was
+        # finished and successful
+
+        self.wait_for(
+            self.get_last_build_number,
+            expected=1,
+            debug_msg='Checking that first build of "petclinic-java-app-deploy"'
+                      ' job is successfully completed',
+            fail_msg='Job "petclinic-java-app-deploy" has failed',
+            timeout=60,
+            ip=fips['org.openstack.ci_cd_pipeline_murano_app.Jenkins'],
+            user=ldap_user,
+            password=ldap_password,
+            job_name='petclinic-java-app-deploy',
+            build_type='lastSuccessfulBuild'
+        )
+
+        # Check that Petclinic application was successfully deployed
+
+        self.wait_for(
+            func=self.check_url_access,
+            expected=200,
+            debug_msg='Checking that "petlinic" app is deployed and available',
+            fail_msg='Petclinic url isn\'t accessible.',
+            timeout=300,
+            ip=fips['com.mirantis.docker.DockerStandaloneHost'],
+            path='petclinic/',
+            port=8080
+        )

tox.ini

@@ -26,6 +26,10 @@ commands = {posargs:}
 commands = python -m unittest tests.test_cicd_apps.MuranoCiCdTest.test_deploy_cicd
 #commands = python setup.py testr --testr-args='{posargs}'
 
+[testenv:run_cicd_flow]
+# FIXME!
+commands = python -m unittest tests.test_cicd_apps_flow.MuranoCiCdFlowTest.test_run_cicd_flow
+
 [testenv:hacking]
 deps=
     ipdb