Add new version of tests
- Created new tests structure - Refactored and partially rewrote code from ugly.py - Added tox.ini Change-Id: I154906b8fcc7870b82fd2b782d03e029d9dd1df2
This commit is contained in:
Anastasia Kuznetsova
alexz
parent
52a4cc6b70
commit
539613ce4f
4
.testr.conf
Normal file
4
.testr.conf
Normal file
@ -0,0 +1,4 @@
|
||||
[DEFAULT]
|
||||
test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./tests -s ${SUBUNIT_TEST_PATH:-./tests/} $LISTOPT $IDOPTION
|
||||
test_id_option=--load-list $IDFILE
|
||||
test_list_option=--list
|
||||
22
setup.cfg
Normal file
22
setup.cfg
Normal file
@ -0,0 +1,22 @@
|
||||
[metadata]
|
||||
name = refstack
|
||||
summary = OpenStack interop testing
|
||||
description-file =
|
||||
README.rst
|
||||
author = OpenStack
|
||||
author-email = openstack-dev@lists.openstack.org
|
||||
home-page = http://www.openstack.org/
|
||||
classifier =
|
||||
Environment :: OpenStack
|
||||
Intended Audience :: Developers
|
||||
Intended Audience :: Information Technology
|
||||
License :: OSI Approved :: Apache Software License
|
||||
Operating System :: POSIX :: Linux
|
||||
Programming Language :: Python
|
||||
Programming Language :: Python :: 2
|
||||
Programming Language :: Python :: 2.7
|
||||
Programming Language :: Python :: 3.3
|
||||
|
||||
[global]
|
||||
setup-hooks =
|
||||
pbr.hooks.setup_hook
|
||||
21
setup.py
Normal file
21
setup.py
Normal file
@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2014 Piston Cloud Computing, inc. all rights reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import setuptools
|
||||
|
||||
setuptools.setup(
|
||||
setup_requires=['pbr'],
|
||||
pbr=True)
|
||||
9
test-requirements.txt
Normal file
9
test-requirements.txt
Normal file
@ -0,0 +1,9 @@
|
||||
testrepository
|
||||
testtools
|
||||
python-heatclient
|
||||
python-keystoneclient
|
||||
requests
|
||||
# FIXME: revert to pip client, when issue with yaql will be fixed
|
||||
#python-muranoclient
|
||||
git+https://github.com/openstack/python-muranoclient.git@master
|
||||
|
||||
0
tests/__init__.py
Executable file
0
tests/__init__.py
Executable file
331
tests/base.py
Executable file
331
tests/base.py
Executable file
@ -0,0 +1,331 @@
|
||||
# Copyright (c) 2016 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import requests
|
||||
import testtools
|
||||
import yaml
|
||||
import muranoclient.common.exceptions as exceptions
|
||||
|
||||
import clients
|
||||
|
||||
ARTIFACTS_DIR = os.environ.get('ARTIFACTS_DIR', 'logs')
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
LOG.setLevel(logging.DEBUG)
|
||||
if not os.path.exists(ARTIFACTS_DIR):
|
||||
os.makedirs(ARTIFACTS_DIR)
|
||||
fh = logging.FileHandler(os.path.join(ARTIFACTS_DIR, 'runner.log'))
|
||||
formatter = logging.Formatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
fh.setFormatter(formatter)
|
||||
LOG.addHandler(fh)
|
||||
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
ch.setFormatter(formatter)
|
||||
LOG.addHandler(ch)
|
||||
|
||||
# Sometimes need to pass some boolean from bash env. Since each bash
|
||||
# variable is string, we need such simply hack
|
||||
_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
|
||||
'0': False, 'no': False, 'false': False, 'off': False}
|
||||
|
||||
|
||||
def str2bool(name, default):
|
||||
value = os.environ.get(name, '')
|
||||
return _boolean_states.get(value.lower(), default)
|
||||
|
||||
|
||||
class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
|
||||
|
||||
def setUp(self):
|
||||
super(MuranoTestsBase, self).setUp()
|
||||
# counter, for murano deployment logger
|
||||
self.latest_report = 0
|
||||
self.flavor = os.environ.get('OS_FLAVOR', 'm1.medium')
|
||||
self.image = os.environ.get('OS_IMAGE')
|
||||
self.keyname = os.environ.get('OS_KEYNAME', None)
|
||||
self.availability_zone = os.environ.get('OS_ZONE', 'nova')
|
||||
self.deploy_timeout = (60 * 60) * 2
|
||||
# Since its really useful to debug deployment after it fail...lets
|
||||
# add such possibility
|
||||
self.os_cleanup_before = str2bool('OS_CLEANUP_BEFORE', False)
|
||||
self.os_cleanup_after = str2bool('OS_CLEANUP_AFTER', True)
|
||||
|
||||
self.keystone = self.initialize_keystone_client()
|
||||
self.heat = self.initialize_heat_client(self.keystone)
|
||||
self.murano = self.initialize_murano_client(self.keystone)
|
||||
self.headers = {
|
||||
'X-Auth-Token': self.murano.http_client.auth_token,
|
||||
'content-type': 'application/json'
|
||||
}
|
||||
self.envs = []
|
||||
if self.os_cleanup_before:
|
||||
self.cleanup_up_tenant()
|
||||
LOG.info('Running test: {0}'.format(self._testMethodName))
|
||||
|
||||
def tearDown(self):
|
||||
if not self.os_cleanup_after:
|
||||
for env in self.envs:
|
||||
try:
|
||||
self.delete_env(env)
|
||||
except Exception:
|
||||
self.delete_stack(env)
|
||||
|
||||
super(MuranoTestsBase, self).tearDown()
|
||||
|
||||
@staticmethod
|
||||
def rand_name(name='murano_ci_test_'):
|
||||
return name + str(time.strftime("%Y_%m_%d_%H_%M_%S"))
|
||||
|
||||
@staticmethod
|
||||
def generate_id():
|
||||
return uuid.uuid4()
|
||||
|
||||
def _get_stack(self, environment_id):
|
||||
for stack in self.heat.stacks.list():
|
||||
if environment_id in stack.description:
|
||||
return stack
|
||||
|
||||
def cleanup_up_tenant(self):
|
||||
LOG.debug('Removing EVERYTHING in tenant: {0}'.format(
|
||||
self.keystone.tenant_name))
|
||||
for env in self.murano.environments.list():
|
||||
self.delete_env(env)
|
||||
for stack in self.heat.stacks.list():
|
||||
try:
|
||||
self.heat.stacks.delete(stack.id)
|
||||
except Exception as e:
|
||||
LOG.warning("Unable delete stack:{}".format(stack))
|
||||
LOG.exception(e)
|
||||
pass
|
||||
return
|
||||
|
||||
def delete_stack(self, environment):
|
||||
stack = self._get_stack(environment.id)
|
||||
if not stack:
|
||||
return
|
||||
else:
|
||||
self.heat.stacks.delete(stack.id)
|
||||
|
||||
def create_env(self):
|
||||
name = self.rand_name()
|
||||
environment = self.murano.environments.create({'name': name})
|
||||
self.envs.append(environment)
|
||||
self.addCleanup(self.delete_env, environment)
|
||||
LOG.debug('Created Environment:\n {0}'.format(environment))
|
||||
|
||||
return environment
|
||||
|
||||
def delete_env(self, environment, timeout=360):
|
||||
try:
|
||||
self.murano.environments.delete(environment.id)
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
try:
|
||||
self.murano.environments.get(environment.id)
|
||||
time.sleep(1)
|
||||
except exceptions.HTTPNotFound:
|
||||
return
|
||||
raise exceptions.HTTPOverLimit(
|
||||
'Environment "{0}" was not deleted in {1} seconds'.format(
|
||||
environment.id, timeout)
|
||||
)
|
||||
except (exceptions.HTTPForbidden, exceptions.HTTPOverLimit,
|
||||
exceptions.HTTPNotFound):
|
||||
try:
|
||||
self.murano.environments.delete(environment.id, abandon=True)
|
||||
LOG.warning(
|
||||
'Environment "{0}" from test {1} abandoned'.format(
|
||||
environment.id, self._testMethodName))
|
||||
except exceptions.HTTPNotFound:
|
||||
return
|
||||
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
try:
|
||||
self.murano.environments.get(environment.id)
|
||||
time.sleep(1)
|
||||
except exceptions.HTTPNotFound:
|
||||
return
|
||||
raise Exception(
|
||||
'Environment "{0}" was not deleted in {1} seconds'.format(
|
||||
environment.id, timeout)
|
||||
)
|
||||
|
||||
def get_env(self, environment):
|
||||
return self.murano.environments.get(environment.id)
|
||||
|
||||
def deploy_env(self, environment, session):
|
||||
self.murano.sessions.deploy(environment.id, session.id)
|
||||
return self.wait_for_environment_deploy(environment)
|
||||
|
||||
def get_deployment_report(self, environment, deployment):
|
||||
history = ''
|
||||
report = self.murano.deployments.reports(environment.id, deployment.id)
|
||||
for status in report:
|
||||
history += '\t{0} - {1}\n'.format(status.created, status.text)
|
||||
return history
|
||||
|
||||
def _log_report(self, environment):
|
||||
deployment = self.murano.deployments.list(environment.id)[0]
|
||||
details = deployment.result['result']['details']
|
||||
LOG.error('Exception found:\n {0}'.format(details))
|
||||
report = self.get_deployment_report(environment, deployment)
|
||||
LOG.debug('Report:\n {0}\n'.format(report))
|
||||
|
||||
def _log_latest(self, environment):
|
||||
deployment = self.murano.deployments.list(environment.id)[0]
|
||||
history = self.get_deployment_report(environment, deployment)
|
||||
if self.latest_report != len(history) or self.latest_report == 0:
|
||||
tmp = len(history)
|
||||
history = history[self.latest_report:]
|
||||
LOG.debug("Last report from murano engine:\n{}".format((history)))
|
||||
self.latest_report = tmp
|
||||
return history
|
||||
|
||||
def wait_for_environment_deploy(self, env):
|
||||
start_time = time.time()
|
||||
status = self.get_env(env).manager.get(env.id).status
|
||||
|
||||
while status != 'ready':
|
||||
status = self.get_env(env).manager.get(env.id).status
|
||||
LOG.debug('Deployment status:{}...nothing new..'.format(status))
|
||||
self._log_latest(env)
|
||||
|
||||
if time.time() - start_time > self.deploy_timeout:
|
||||
time.sleep(60)
|
||||
self.fail(
|
||||
'Environment deployment wasn\'t'
|
||||
'finished in {} seconds'.format(self.deploy_timeout)
|
||||
)
|
||||
elif status == 'deploy failure':
|
||||
self._log_report(env)
|
||||
self.fail(
|
||||
'Environment has incorrect status "{0}"'.format(status)
|
||||
)
|
||||
|
||||
time.sleep(30)
|
||||
LOG.debug('Environment "{0}" is ready'.format(self.get_env(env).name))
|
||||
return self.get_env(env).manager.get(env.id)
|
||||
|
||||
def create_session(self, environment):
|
||||
return self.murano.sessions.configure(environment.id)
|
||||
|
||||
def create_service(self, environment, session, json_data, to_json=True):
|
||||
LOG.debug('Adding service:\n {0}'.format(json_data))
|
||||
service = self.murano.services.post(
|
||||
environment.id,
|
||||
path='/',
|
||||
data=json_data,
|
||||
session_id=session.id
|
||||
)
|
||||
if to_json:
|
||||
service = service.to_dict()
|
||||
service = json.dumps(service)
|
||||
LOG.debug('Create Service json: {0}'.format(yaml.load(service)))
|
||||
return yaml.load(service)
|
||||
else:
|
||||
LOG.debug('Create Service: {0}'.format(service))
|
||||
return service
|
||||
|
||||
@staticmethod
|
||||
def guess_fip(env_obj_model):
|
||||
|
||||
result = {}
|
||||
|
||||
def _finditem(obj, result):
|
||||
if 'floatingIpAddress' in obj.get('instance', []):
|
||||
result[obj['?']['package']] = obj['instance'][
|
||||
'floatingIpAddress']
|
||||
for k, v in obj.items():
|
||||
if isinstance(v, dict):
|
||||
_finditem(v, result)
|
||||
_finditem(env_obj_model, result)
|
||||
|
||||
return result
|
||||
|
||||
def check_ports_open(self, ip, ports):
|
||||
for port in ports:
|
||||
result = 1
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < 60:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
result = sock.connect_ex((str(ip), port))
|
||||
sock.close()
|
||||
|
||||
if result == 0:
|
||||
LOG.debug('{} port is opened on instance'.format(port))
|
||||
break
|
||||
time.sleep(5)
|
||||
|
||||
if result != 0:
|
||||
self.fail('{} port is not opened on instance'.format(port))
|
||||
|
||||
def check_url_access(self, ip, path, port):
|
||||
attempt = 0
|
||||
proto = 'http' if port not in (443, 8443) else 'https'
|
||||
url = '%s://%s:%s/%s' % (proto, ip, port, path)
|
||||
|
||||
while attempt < 5:
|
||||
resp = requests.get(url)
|
||||
if resp.status_code == 200:
|
||||
LOG.debug('Service path "{}" is available'.format(url))
|
||||
return
|
||||
else:
|
||||
time.sleep(5)
|
||||
attempt += 1
|
||||
|
||||
self.fail(
|
||||
'Service path {0} is unavailable after 5 attempts'.format(url)
|
||||
)
|
||||
|
||||
def deployment_success_check(self, environment, services_map):
|
||||
deployment = self.murano.deployments.list(environment.id)[-1]
|
||||
|
||||
self.assertEqual(
|
||||
'success', deployment.state,
|
||||
'Deployment status is "{0}"'.format(deployment.state)
|
||||
)
|
||||
|
||||
fips = self.guess_fip(environment.services[0])
|
||||
|
||||
for service in services_map:
|
||||
LOG.debug(
|
||||
'Checking ports availability on "{}" app instance'.format(
|
||||
service)
|
||||
)
|
||||
self.check_ports_open(
|
||||
fips[service], services_map[service]['ports']
|
||||
)
|
||||
if services_map[service]['url']:
|
||||
LOG.debug(
|
||||
'Checking {0} app url "{1}" availability'.format(
|
||||
service, services_map[service]['url']
|
||||
)
|
||||
)
|
||||
self.check_url_access(
|
||||
fips[service],
|
||||
services_map[service]['url'],
|
||||
services_map[service]['url_port']
|
||||
)
|
||||
84
tests/clients.py
Executable file
84
tests/clients.py
Executable file
@ -0,0 +1,84 @@
|
||||
# Copyright (c) 2016 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from heatclient import client as heatclient
|
||||
from keystoneclient.v2_0 import client as keystoneclient
|
||||
from muranoclient import client as muranoclient
|
||||
|
||||
|
||||
class ClientsBase(object):
|
||||
|
||||
@staticmethod
|
||||
def initialize_keystone_client():
|
||||
username = os.environ.get('OS_USERNAME')
|
||||
password = os.environ.get('OS_PASSWORD')
|
||||
tenant_name = os.environ.get('OS_TENANT_NAME')
|
||||
os_auth_uri = os.environ.get('OS_AUTH_URL')
|
||||
|
||||
keystone = keystoneclient.Client(
|
||||
username=username,
|
||||
password=password,
|
||||
tenant_name=tenant_name,
|
||||
auth_url=os_auth_uri
|
||||
)
|
||||
return keystone
|
||||
|
||||
@classmethod
|
||||
def get_endpoint(cls, service_type, endpoint_type):
|
||||
ks_client = cls.initialize_keystone_client()
|
||||
|
||||
return ks_client.service_catalog.url_for(
|
||||
service_type=service_type,
|
||||
endpoint_type=endpoint_type
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def initialize_murano_client(cls, auth_client=None):
|
||||
ks_client = (auth_client if auth_client
|
||||
else cls.initialize_keystone_client())
|
||||
|
||||
murano_endpoint = cls.get_endpoint(
|
||||
service_type='application-catalog',
|
||||
endpoint_type='publicURL'
|
||||
)
|
||||
|
||||
murano = muranoclient.Client(
|
||||
'1',
|
||||
endpoint=murano_endpoint,
|
||||
token=ks_client.auth_token
|
||||
)
|
||||
|
||||
return murano
|
||||
|
||||
@classmethod
|
||||
def initialize_heat_client(cls, auth_client=None):
|
||||
ks_client = (auth_client if auth_client
|
||||
else cls.initialize_keystone_client())
|
||||
|
||||
heat_endpoint = cls.get_endpoint(
|
||||
service_type='orchestration',
|
||||
endpoint_type='publicURL'
|
||||
)
|
||||
|
||||
heat = heatclient.Client(
|
||||
'1',
|
||||
endpoint=heat_endpoint,
|
||||
token=ks_client.auth_token
|
||||
)
|
||||
|
||||
return heat
|
||||
|
||||
70
tests/test_cicd_apps.py
Executable file
70
tests/test_cicd_apps.py
Executable file
@ -0,0 +1,70 @@
|
||||
# Copyright (c) 2016 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import base
|
||||
|
||||
|
||||
class MuranoCiCdTest(base.MuranoTestsBase):
|
||||
|
||||
def test_deploy_cicd(self):
|
||||
environment = self.create_env()
|
||||
session = self.create_session(environment)
|
||||
service_json = {
|
||||
'?': {
|
||||
'_{id}'.format(id=self.generate_id().hex): {'name': 'CI/CD'},
|
||||
'id': str(self.generate_id()),
|
||||
'type':
|
||||
'org.openstack.ci_cd_pipeline_murano_app.CiCdEnvironment'
|
||||
},
|
||||
'assignFloatingIp': True,
|
||||
'availabilityZone': self.availability_zone,
|
||||
'flavor': self.flavor,
|
||||
'image': self.image,
|
||||
'instance_name': environment.name,
|
||||
'keyname': self.keyname,
|
||||
'ldapEmail': 'email@example.com',
|
||||
'ldapPass': 'P@ssw0rd',
|
||||
'ldapRootEmail': 'root@example.com',
|
||||
'ldapRootPass': 'P@ssw0rd',
|
||||
'ldapRootUser': 'root',
|
||||
'ldapUser': 'user',
|
||||
'name': 'CI/CD'
|
||||
}
|
||||
self.create_service(environment, session, service_json)
|
||||
self.deploy_env(environment, session)
|
||||
|
||||
environment = self.get_env(environment)
|
||||
check_services = {
|
||||
'org.openstack.ci_cd_pipeline_murano_app.Jenkins': {
|
||||
'ports': [8080, 22],
|
||||
'url': 'api/',
|
||||
'url_port': 8080
|
||||
},
|
||||
'org.openstack.ci_cd_pipeline_murano_app.Gerrit': {
|
||||
'ports': [8081, 22],
|
||||
'url': '#/admin/projects/',
|
||||
'url_port': 8081
|
||||
},
|
||||
'org.openstack.ci_cd_pipeline_murano_app.OpenLDAP': {
|
||||
'ports': [389, 22],
|
||||
'url': None
|
||||
},
|
||||
}
|
||||
|
||||
self.deployment_success_check(environment, check_services)
|
||||
base.LOG.debug("Run second deployment, w\o any changes..")
|
||||
session = self.create_session(environment)
|
||||
self.deploy_env(environment, session)
|
||||
self.deployment_success_check(environment, check_services)
|
||||
@ -1,34 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
# Requirments:
|
||||
# apt-get install python-venv python-pip build-essential libssl-dev libffi-dev -y
|
||||
|
||||
# Variables:
|
||||
VENV_PATH=${VENV_PATH:-.venv}
|
||||
VENV_CLEAN=${VENV_CLEAN:-false}
|
||||
TEST_NAME=${TEST_NAME:-none}
|
||||
|
||||
function prepare_venv() {
|
||||
echo 'LOG: Creating python venv for murano-client'
|
||||
rm -rf "${VENV_PATH}"
|
||||
mkdir -p "${VENV_PATH}"
|
||||
virtualenv --system-site-packages "${VENV_PATH}"
|
||||
source "${VENV_PATH}/bin/activate"
|
||||
#TODO install from requirments.txt ?
|
||||
pip install python-muranoclient python-heatclient
|
||||
deactivate
|
||||
}
|
||||
|
||||
# Body
|
||||
if [[ ("${VENV_CLEAN}" == true) || (! -f "${VENV_PATH}/bin/activate") ]]; then
|
||||
prepare_venv
|
||||
fi
|
||||
|
||||
if [[ "${TEST_NAME}" != "none" ]] ; then
|
||||
source "${VENV_PATH}/bin/activate"
|
||||
echo "LOG: Attempt to run test=${TEST_NAME}"
|
||||
./utils/jenkins/"${TEST_NAME}"
|
||||
deactivate
|
||||
fi
|
||||
|
||||
@ -1,384 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Ugly script, for quick deployment. copy-paste from
|
||||
# https://github.com/vryzhenkin/pegasus
|
||||
|
||||
from muranoclient import client as muranocl
|
||||
import os
|
||||
import logging
|
||||
import urlparse
|
||||
import yaml
|
||||
import json
|
||||
import uuid
|
||||
import time
|
||||
import socket
|
||||
import pprint
|
||||
import requests
|
||||
from keystoneclient.v2_0 import client as keystoneclient
|
||||
from heatclient.client import Client as HeatCl
|
||||
|
||||
username = os.environ.get('OS_USERNAME')
|
||||
password = os.environ.get('OS_PASSWORD')
|
||||
tenant_name = os.environ.get('OS_TENANT_NAME', False)
|
||||
uri = os.environ.get('OS_AUTH_URL')
|
||||
murano_endpoint = os.environ.get('OS_MURANO_URL')
|
||||
heat_endpoint = os.environ.get('OS_HEAT_URL')
|
||||
|
||||
os_cleanup = os.environ.get('OS_CLEANUP', False)
|
||||
murano_env_name = os.environ.get('ENV_NAME_PREFIX', 'murano_ci_test_')
|
||||
flavor = os.environ.get('OS_FLAVOR', 'm1.medium')
|
||||
image = os.environ.get('OS_IMAGE', 'Ubuntu_14.04_x64_murano-agent')
|
||||
keyname = os.environ.get('OS_KEYNAME', 'test_key')
|
||||
availability_zone = os.environ.get('OS_ZONE', 'nova')
|
||||
# to be passed to murano-app
|
||||
m_pass = 'P@ssw0rd'
|
||||
ARTIFACTS_DIR = os.environ.get('ARTIFACTS_DIR', 'logs')
|
||||
BUILD_TAG = os.environ.get('BUILD_TAG', None)
|
||||
deploy_timeout = (60 * 60) * 2
|
||||
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - '
|
||||
'%(levelname)s - %(message)s')
|
||||
LOG = logging.getLogger(__name__)
|
||||
LOG.setLevel(logging.DEBUG)
|
||||
if not os.path.exists(ARTIFACTS_DIR):
|
||||
os.makedirs(ARTIFACTS_DIR)
|
||||
fh = logging.FileHandler(os.path.join(ARTIFACTS_DIR, 'runner.log'))
|
||||
fh.setLevel(logging.DEBUG)
|
||||
fh.setFormatter(formatter)
|
||||
LOG.addHandler(fh)
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
ch.setFormatter(formatter)
|
||||
LOG.addHandler(ch)
|
||||
pprinter = pprint.PrettyPrinter(indent=1, width=80, depth=None)
|
||||
history_log = 0
|
||||
|
||||
murano_client = None
|
||||
|
||||
# service name , from murano-app map, and port, which should be checked
|
||||
check_map = {'org.openstack.ci_cd_pipeline_murano_app.Jenkins':
|
||||
{'ports': [8080, 22], 'url': 'api/', 'url_port': 8080},
|
||||
'org.openstack.ci_cd_pipeline_murano_app.Gerrit':
|
||||
{'ports': [8081, 22], 'url': '#/admin/projects/',
|
||||
'url_port': 8081},
|
||||
'org.openstack.ci_cd_pipeline_murano_app.OpenLDAP':
|
||||
{'ports': [389, 22], 'url': None},
|
||||
}
|
||||
|
||||
|
||||
def get_murano_client():
|
||||
"""
|
||||
Hook for keystone
|
||||
:return:
|
||||
"""
|
||||
global murano_client
|
||||
if murano_client is None:
|
||||
keystone = get_auth()
|
||||
murano_client = muranocl.Client('1', endpoint=murano_endpoint,
|
||||
token=keystone.auth_token)
|
||||
return murano_client
|
||||
try:
|
||||
keystoneclient.Client(token=murano_client.http_client.auth_token,
|
||||
auth_url=uri)
|
||||
except Exception as e:
|
||||
keystone = get_auth()
|
||||
murano_client = muranocl.Client('1', endpoint=murano_endpoint,
|
||||
token=keystone.auth_token)
|
||||
return murano_client
|
||||
|
||||
|
||||
def get_auth():
|
||||
keystone = keystoneclient.Client(username=username,
|
||||
password=password,
|
||||
tenant_name=tenant_name,
|
||||
auth_url=uri)
|
||||
return keystone
|
||||
|
||||
|
||||
def rand_name(name=murano_env_name):
|
||||
return name + str(time.strftime("%Y_%m_%d_%H_%M_%S"))
|
||||
|
||||
|
||||
def create_env():
|
||||
if BUILD_TAG is None:
|
||||
name = rand_name(murano_env_name)
|
||||
else:
|
||||
name = murano_env_name + str(BUILD_TAG)
|
||||
environment = get_murano_client().environments.create({'name': name})
|
||||
LOG.debug(
|
||||
'Created Environment:\n{0}'.format(pprinter.pformat(environment)))
|
||||
return environment.id
|
||||
|
||||
|
||||
def get_env(env_id=None):
|
||||
if env_id:
|
||||
return get_murano_client().environments.get(env_id)
|
||||
else:
|
||||
fail('Wrong environment id!')
|
||||
|
||||
|
||||
def add_service(env_id, data, session):
|
||||
"""
|
||||
This function adding a specific service to environment
|
||||
Returns a specific class <Service>
|
||||
:param env_id:
|
||||
:param data:
|
||||
:param session:
|
||||
:return:
|
||||
"""
|
||||
LOG.debug('Added service:\n {0}'.format(data))
|
||||
return get_murano_client().services.post(env_id,
|
||||
path='/',
|
||||
data=data,
|
||||
session_id=session.id)
|
||||
|
||||
|
||||
def create_service(env_id, session, json_data, to_json=True):
|
||||
"""
|
||||
This function adding a specific service to environment
|
||||
Returns a JSON object with a service
|
||||
:param env_id:
|
||||
:param session:
|
||||
:param json_data:
|
||||
:return:
|
||||
"""
|
||||
|
||||
service = add_service(env_id, json_data, session)
|
||||
if to_json:
|
||||
service = service.to_dict()
|
||||
service = json.dumps(service)
|
||||
LOG.debug('Create Service json:{0}'.format(pprinter.pformat(service)))
|
||||
return yaml.load(service)
|
||||
else:
|
||||
LOG.debug('Create Service: {0}'.format(service))
|
||||
return service
|
||||
|
||||
|
||||
def fail(msg=None):
|
||||
"""Fail immediately, with the given message."""
|
||||
raise Exception(msg)
|
||||
|
||||
|
||||
def get_last_deployment(env_id):
|
||||
deployments = get_murano_client().deployments.list(env_id)
|
||||
return deployments[0]
|
||||
|
||||
|
||||
def get_deployment_report(env_id, deployment, as_list=False):
|
||||
report = get_murano_client().deployments.reports(env_id, deployment.id)
|
||||
if as_list:
|
||||
history = []
|
||||
for status in report:
|
||||
history.append('{0} - {1}'.format(status.created, status.text))
|
||||
else:
|
||||
history = ''
|
||||
for status in report:
|
||||
history += '\t{0} - {1}\n'.format(status.created, status.text)
|
||||
return history
|
||||
|
||||
|
||||
def _log_report(env_id):
|
||||
deployment = get_last_deployment(env_id)
|
||||
try:
|
||||
details = deployment.result['result']['details']
|
||||
except KeyError:
|
||||
LOG.error('Deployment has no result details!')
|
||||
pass
|
||||
LOG.error('Exception found:\n {0}'.format(details))
|
||||
report = get_deployment_report(env_id, deployment)
|
||||
LOG.debug('Report:\n {0}\n'.format(report))
|
||||
|
||||
|
||||
def _log_quick(env_id):
|
||||
global history_log
|
||||
deployment = get_last_deployment(env_id)
|
||||
history = get_deployment_report(env_id, deployment, as_list=True)
|
||||
if history_log != len(history) or history_log == 0:
|
||||
tmp = len(history)
|
||||
history = history[history_log:]
|
||||
LOG.debug("Last report:\n{}".format(pprinter.pformat(history)))
|
||||
history_log = tmp
|
||||
return history
|
||||
|
||||
|
||||
def wait_for_environment_deploy(env_id):
|
||||
start_time = time.time()
|
||||
status = get_env(env_id).manager.get(env_id).status
|
||||
while status != 'ready':
|
||||
status = get_env(env_id).manager.get(env_id).status
|
||||
LOG.debug('Deployment status:{}...nothing new..'.format(status))
|
||||
_log_quick(env_id)
|
||||
if time.time() - start_time > deploy_timeout:
|
||||
time.sleep(60)
|
||||
_log_report(env_id)
|
||||
fail(
|
||||
'Environment deployment is not finished in {}seconds'.format(
|
||||
deploy_timeout))
|
||||
elif status == 'deploy failure':
|
||||
_log_report(env_id)
|
||||
fail('Environment has incorrect status {0}'.format(status))
|
||||
time.sleep(30)
|
||||
LOG.debug('Environment {0} is ready'.format(get_env(env_id).name))
|
||||
return get_env(env_id).manager.get(env_id)
|
||||
|
||||
|
||||
def deploy_environment(env_id, session):
|
||||
get_murano_client().sessions.deploy(env_id, session.id)
|
||||
return wait_for_environment_deploy(env_id)
|
||||
|
||||
|
||||
def divine_fip(obj):
|
||||
"""
|
||||
will return dict like app_type : fip
|
||||
|
||||
:param obj:
|
||||
:param result:
|
||||
:return:
|
||||
"""
|
||||
result = {}
|
||||
|
||||
def _finditem(obj, result):
|
||||
if 'floatingIpAddress' in obj.get('instance', []):
|
||||
result[obj['?']['package']] = obj['instance'][
|
||||
'floatingIpAddress']
|
||||
for k, v in obj.items():
|
||||
if isinstance(v, dict):
|
||||
_finditem(v, result)
|
||||
_finditem(obj, result)
|
||||
return result
|
||||
|
||||
|
||||
def check_port_access(ip, port):
|
||||
# FIXME
|
||||
result = 1
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < 60:
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
result = sock.connect_ex((str(ip), port))
|
||||
sock.close()
|
||||
if result == 0:
|
||||
LOG.debug('%s port is opened on instance' % port)
|
||||
break
|
||||
else:
|
||||
fail('%s port is not opened on instance' % port)
|
||||
time.sleep(5)
|
||||
if result != 0:
|
||||
fail('%s port is not opened on instance' % port)
|
||||
|
||||
|
||||
def check_path(path, ip, port=80):
|
||||
attempts = 5
|
||||
proto = 'http'
|
||||
if port in (443, 8443):
|
||||
proto = 'https'
|
||||
url = '%s://%s:%s/%s' % (proto, ip, port, path)
|
||||
for i in range(attempts):
|
||||
resp = requests.get(url)
|
||||
i += 1
|
||||
if resp.status_code != 200 and i >= attempts:
|
||||
fail('Service path failed: %s Code:%s' % (url, resp.status_code))
|
||||
elif resp.status_code == 200:
|
||||
LOG.debug('Service path fine: %s' % url)
|
||||
return
|
||||
else:
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def deployment_success_check(env_id, ip, inst_name='jenkins', ports=[]):
|
||||
deployment = get_murano_client().deployments.list(env_id)[-1]
|
||||
LOG.debug('Deployment status is {0}'.format(deployment.state))
|
||||
if str(deployment.state) != 'success':
|
||||
fail('Wrong deploymnet state = {}'.format(deployment.state))
|
||||
for port in ports:
|
||||
LOG.debug("Looking into: {} {}:{} ".format(inst_name, ip, port))
|
||||
check_port_access(ip, port)
|
||||
|
||||
|
||||
def cleaup_up_tenant():
|
||||
LOG.warning('Removing everything from tenant{}'.format(tenant_name))
|
||||
murano = get_murano_client()
|
||||
for env in murano.environments.list():
|
||||
try:
|
||||
murano.environments.delete(env.id)
|
||||
except Exception as e:
|
||||
LOG.warning("Unable delete env:{}".format(env.id))
|
||||
LOG.exception(e)
|
||||
try:
|
||||
LOG.warning("Trying abandon env:{}".format(env.id))
|
||||
murano.environments.delete(env.id, abandon=True)
|
||||
except Exception as e:
|
||||
LOG.warning("Unable abandon env:{}".format(env.id))
|
||||
LOG.exception(e)
|
||||
pass
|
||||
pass
|
||||
|
||||
tenant_id = get_auth().get_project_id(tenant_name)
|
||||
heat_url = urlparse.urljoin(heat_endpoint, tenant_id)
|
||||
heat = HeatCl('1', endpoint=heat_url, token=get_auth().auth_token)
|
||||
for stack in heat.stacks.list():
|
||||
try:
|
||||
heat.stacks.delete(stack.id)
|
||||
except Exception as e:
|
||||
LOG.warning("Unable delete stack:{}".format(stack))
|
||||
LOG.exception(e)
|
||||
pass
|
||||
return
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
if os_cleanup and tenant_name.lower() == 'admin':
|
||||
fail(
|
||||
"Never use this ugly test with 'admin' tenant! it can "
|
||||
"destroy to much!")
|
||||
|
||||
if os_cleanup:
|
||||
LOG.warning('Removing all stuff')
|
||||
cleaup_up_tenant()
|
||||
|
||||
# instant test
|
||||
# murano = get_murano_client()
|
||||
# environment_id = murano.environments.get('deca41a0e8504eef864')
|
||||
|
||||
env_id = create_env()
|
||||
session = get_murano_client().sessions.configure(env_id)
|
||||
|
||||
post_body = {
|
||||
'?': {'_{id}'.format(id=uuid.uuid4().hex): {'name': 'CI/CD'},
|
||||
'id': str(uuid.uuid4()),
|
||||
'type': 'org.openstack.ci_cd_pipeline_murano_app.CiCdEnvironment'},
|
||||
'assignFloatingIp': True,
|
||||
'availabilityZone': availability_zone,
|
||||
'flavor': flavor,
|
||||
'image': image,
|
||||
'instance_name': get_env(env_id).name,
|
||||
'keyname': keyname,
|
||||
'ldapEmail': 'email@example.com',
|
||||
'ldapPass': m_pass,
|
||||
'ldapRootEmail': 'root@example.com',
|
||||
'ldapRootPass': m_pass,
|
||||
'ldapRootUser': 'root',
|
||||
'ldapUser': 'user',
|
||||
'name': 'CI/CD'
|
||||
}
|
||||
|
||||
try:
|
||||
create_service(env_id, session, post_body)
|
||||
LOG.debug("Attempt to deploy env..")
|
||||
deploy_environment(env_id, session)
|
||||
fip_map = divine_fip(get_env(env_id).services[0])
|
||||
for app in check_map:
|
||||
deployment_success_check(env_id, fip_map[app], app,
|
||||
check_map[app]['ports'])
|
||||
if check_map[app]['url']:
|
||||
LOG.debug('Checking service {}'.format(app))
|
||||
check_path(check_map[app]['url'], fip_map[app],
|
||||
port=check_map[app]['url_port'])
|
||||
LOG.debug("Deployment finished successfully")
|
||||
|
||||
except Exception as exc:
|
||||
LOG.exception('Deployment error %s' % exc)
|
||||
raise
|
||||
finally:
|
||||
deployment = get_last_deployment(env_id)
|
||||
|
||||
31
tox.ini
Normal file
31
tox.ini
Normal file
@ -0,0 +1,31 @@
|
||||
[tox]
|
||||
minversion = 1.6
|
||||
skipsdist = True
|
||||
envlist = py27
|
||||
skip_missing_interpreters = True
|
||||
|
||||
[testenv]
|
||||
setenv = VIRTUAL_ENV={envdir}
|
||||
LANG=en_US.UTF-8
|
||||
LANGUAGE=en_US:en
|
||||
LC_ALL=C
|
||||
passenv = OS_* MURANO* *ENDPOINT*
|
||||
deps=
|
||||
pytz
|
||||
extras
|
||||
python-subunit
|
||||
debtcollector
|
||||
-r{toxinidir}/test-requirements.txt
|
||||
distribute = false
|
||||
|
||||
[testenv:venv]
|
||||
commands = {posargs:}
|
||||
|
||||
[testenv:deploy_cicd_apps]
|
||||
commands = python setup.py testr --testr-args='{posargs}'
|
||||
|
||||
[testenv:hacking]
|
||||
deps=
|
||||
ipdb
|
||||
-r{toxinidir}/test-requirements.txt
|
||||
commands = python -m unittest tests.test_cicd_apps.MuranoCiCdTest.test_deploy_cicd
|
||||
Loading…
x
Reference in New Issue
Block a user