diff --git a/bin/clean_installers.py b/bin/clean_installers.py new file mode 100755 index 00000000..a2fef6fe --- /dev/null +++ b/bin/clean_installers.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Scripts to delete cluster and it hosts""" +import logging +import os +import os.path +import sys + + +current_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(current_dir) + + +import switch_virtualenv + +from compass.actions import clean +from compass.db.api import adapter_holder as adapter_api +from compass.db.api import database +from compass.db.api import user as user_api +from compass.tasks.client import celery +from compass.utils import flags +from compass.utils import logsetting +from compass.utils import setting_wrapper as setting + + +flags.add_bool('async', + help='run in async mode', + default=True) + +flags.add('os_installers', + help='comma seperated os installers', + default='') +flags.add('package_installers', + help='comma separated package installers', + default='') + + +def clean_installers(): + os_installers = [ + os_installer + for os_installer in flags.OPTIONS.os_installers.split(',') + if os_installer + ] + package_installers = [ + package_installer + for package_installer in flags.OPTIONS.package_installers.split(',') + if package_installer + ] + user = user_api.get_user_object(setting.COMPASS_ADMIN_EMAIL) + adapters = adapter_api.list_adapters(user) + filtered_os_installers = {} + filtered_package_installers = {} + for adapter in adapters: + logging.info( + 'got adapter: %s', adapter + ) + if 'os_installer' in adapter: + os_installer = adapter['os_installer'] + os_installer_name = os_installer['alias'] + if not os_installers or os_installer_name in os_installers: + filtered_os_installers[os_installer_name] = os_installer + else: + logging.info( + 'ignore os isntaller %s', os_installer_name + ) + else: + logging.info( + 'cannot find os installer in adapter %s', + adapter['name'] + ) + if 'package_installer' in adapter: + package_installer = adapter['package_installer'] + package_installer_name = package_installer['alias'] + if ( + not package_installers or + package_installer_name in package_installers + ): + filtered_package_installers[package_installer_name] = ( + package_installer + ) + else: + logging.info( + 'ignore package installer %s', package_installer_name + ) + else: + logging.info( + 'cannot find package installer in adapter %s', + adapter['name'] + ) + logging.info( + 'clean os installers: %s', filtered_os_installers.keys() + ) + logging.info( + 'clean package installers: %s', filtered_package_installers.keys() + ) + if flags.OPTIONS.async: + for os_installer_name, os_installer in filtered_os_installers.items(): + celery.send_task( + 'compass.tasks.clean_os_installer', + ( + os_installer['name'], + os_installer['settings'] + ) + ) + for package_installer_name, package_installer in ( + filtered_package_installers.items() + ): + celery.send_task( + 'compass.tasks.clean_package_installer', + ( + package_installer['name'], + package_installer['settings'] + ) + ) + else: + for os_installer_name, os_installer in ( + filtered_os_installers.items() + ): + try: + clean.clean_os_installer( + os_installer['name'], + os_installer['settings'] + ) + except Exception as error: + logging.error( + 'failed to clean os installer %s', os_installer_name + ) + logging.exception(error) + for package_installer_name, package_installer in ( + filtered_package_installers.items() + ): + try: + clean.clean_package_installer( + package_installer['name'], + package_installer['settings'] + ) + except Exception as error: + logging.error( + 'failed to clean package installer %s', + package_installer_name + ) + logging.exception(error) + + +if __name__ == '__main__': + flags.init() + logsetting.init() + database.init() + clean_installers() diff --git a/bin/delete_clusters.py b/bin/delete_clusters.py index 277daed9..c822a4f1 100755 --- a/bin/delete_clusters.py +++ b/bin/delete_clusters.py @@ -56,8 +56,11 @@ def delete_clusters(): if clustername ] user = user_api.get_user_object(setting.COMPASS_ADMIN_EMAIL) + list_cluster_args = {} + if clusternames: + list_cluster_args['name'] = clusternames clusters = cluster_api.list_clusters( - user, name=clusternames + user, **list_cluster_args ) delete_underlying_host = flags.OPTIONS.delete_hosts for cluster in clusters: diff --git a/bin/refresh.sh b/bin/refresh.sh index b2c5ac84..3b992b39 100755 --- a/bin/refresh.sh +++ b/bin/refresh.sh @@ -2,10 +2,7 @@ set -e service mysqld restart /opt/compass/bin/manage_db.py createdb -echo "You may run '/opt/compass/bin/clean_nodes.sh' to clean nodes on chef server" -echo "You may run '/opt/compass/bin/clean_clients.sh' to clean clients on chef server" -echo "you may run '/opt/compass/bin/clean_environments.sh' to clean environments on chef server" -echo "you may run '/opt/compass/bin/remove_systems.sh' to clean systems on cobbler" +/opt/compass/bin/clean_installers.py /opt/compass/bin/clean_installation_logs.py service httpd restart service rsyslog restart diff --git a/compass/actions/clean.py b/compass/actions/clean.py new file mode 100644 index 00000000..c9a5d06c --- /dev/null +++ b/compass/actions/clean.py @@ -0,0 +1,182 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to clean installers +""" +import chef +import logging +import xmlrpclib + +from compass.actions import util + + +class CobblerInstaller(object): + """cobbler installer""" + CREDENTIALS = "credentials" + USERNAME = 'username' + PASSWORD = 'password' + + INSTALLER_URL = "cobbler_url" + + def __init__(self, settings): + username = settings[self.CREDENTIALS][self.USERNAME] + password = settings[self.CREDENTIALS][self.PASSWORD] + cobbler_url = settings[self.INSTALLER_URL] + try: + self.remote = xmlrpclib.Server(cobbler_url) + self.token = self.remote.login(username, password) + logging.info('cobbler %s client created', cobbler_url) + except Exception as error: + logging.error( + 'failed to login %s with (%s, %s)', + cobbler_url, username, password + ) + logging.exception(error) + + def clean(self): + systems = self.remote.get_systems() + for system in systems: + system_name = system['name'] + try: + self.remote.remove_system(system_name, self.token) + logging.info('system %s is removed', system_name) + except Exception as error: + logging.error( + 'failed to remove system %s', system_name + ) + logging.exception(error) + + +class ChefInstaller(object): + DATABAGS = "databags" + CHEFSERVER_URL = "chef_url" + CHEFSERVER_DNS = "chef_server_dns" + CHEFSERVER_IP = "chef_server_ip" + KEY_DIR = "key_dir" + CLIENT = "client_name" + + def __init__(self, settings): + installer_url = settings.get(self.CHEFSERVER_URL, None) + key_dir = settings.get(self.KEY_DIR, None) + client = settings.get(self.CLIENT, None) + try: + if installer_url and key_dir and client: + self.api = chef.ChefAPI(installer_url, key_dir, client) + else: + self.api = chef.autoconfigure() + logging.info( + 'chef client created %s(%s, %s)', + installer_url, key_dir, client + ) + except Exception as error: + logging.error( + 'failed to create chef client %s(%s, %s)', + installer_url, key_dir, client + ) + logging.exception(error) + + def clean(self): + try: + for node_name in chef.Node.list(api=self.api): + node = chef.Node(node_name, api=self.api) + node.delete() + logging.info('delete node %s', node_name) + except Exception as error: + logging.error('failed to delete some nodes') + logging.exception(error) + + try: + for client_name in chef.Client.list(api=self.api): + if client_name in ['chef-webui', 'chef-validator']: + continue + client = chef.Client(client_name, api=self.api) + client.delete() + logging.info('delete client %s', client_name) + except Exception as error: + logging.error('failed to delete some clients') + logging.exception(error) + + try: + for env_name in chef.Environment.list(api=self.api): + if env_name == '_default': + continue + env = chef.Environment(env_name, api=self.api) + env.delete() + logging.info('delete env %s', env_name) + except Exception as error: + logging.error('failed to delete some envs') + logging.exception(error) + + try: + for databag_name in chef.DataBag.list(api=self.api): + databag = chef.DataBag(databag_name, api=self.api) + for item_name, item in databag.items(): + item.delete() + logging.info( + 'delete item %s from databag %s', + item_name, databag_name + ) + except Exception as error: + logging.error('failed to delete some databag items') + logging.exception(error) + + +OS_INSTALLERS = { + 'cobbler': CobblerInstaller +} +PK_INSTALLERS = { + 'chef_installer': ChefInstaller +} + + +def clean_os_installer( + os_installer_name, os_installer_settings +): + with util.lock('serialized_action', timeout=100) as lock: + if not lock: + raise Exception( + 'failed to acquire lock to clean os installer' + ) + + if os_installer_name not in OS_INSTALLERS: + logging.error( + '%s not found in os_installers', + os_installer_name + ) + + os_installer = OS_INSTALLERS[os_installer_name]( + os_installer_settings + ) + os_installer.clean() + + +def clean_package_installer( + package_installer_name, package_installer_settings +): + with util.lock('serialized_action', timeout=100) as lock: + if not lock: + raise Exception( + 'failed to acquire lock to clean package installer' + ) + + if package_installer_name not in PK_INSTALLERS: + logging.error( + '%s not found in os_installers', + package_installer_name + ) + + package_installer = PK_INSTALLERS[package_installer_name]( + package_installer_settings + ) + package_installer.clean() diff --git a/compass/db/api/metadata.py b/compass/db/api/metadata.py index dfa0813f..41122d2b 100644 --- a/compass/db/api/metadata.py +++ b/compass/db/api/metadata.py @@ -301,17 +301,28 @@ def _validate_self( metadata, whole_check, **kwargs ): + logging.debug('validate config self %s', config_path) if '_self' not in metadata: if isinstance(config, dict): _validate_config( config_path, config, metadata, whole_check, **kwargs ) return - field_type = metadata['_self'].get('field_type', 'basestring') + field_type = metadata['_self'].get('field_type', basestring) if not isinstance(config, field_type): raise exception.InvalidParameter( '%s config type is not %s' % (config_path, field_type) ) + is_required = metadata['_self'].get( + 'is_required', False + ) + required_in_whole_config = metadata['_self'].get( + 'required_in_whole_config', False + ) + if isinstance(config, basestring): + if config == '' and not is_required and not required_in_whole_config: + # ignore empty config when it is optional + return required_in_options = metadata['_self'].get( 'required_in_options', False ) @@ -333,6 +344,7 @@ def _validate_self( '%s config is not in %s' % (config_path, options) ) validator = metadata['_self'].get('validator', None) + logging.debug('validate by validator %s', validator) if validator: if not validator(config_key, config, **kwargs): raise exception.InvalidParameter( @@ -348,6 +360,7 @@ def _validate_config( config_path, config, metadata, whole_check, **kwargs ): + logging.debug('validate config %s', config_path) generals = {} specified = {} for key, value in metadata.items(): diff --git a/compass/db/validator.py b/compass/db/validator.py index c8c4d3a2..730bb52c 100644 --- a/compass/db/validator.py +++ b/compass/db/validator.py @@ -13,6 +13,7 @@ # limitations under the License. """Validator methods.""" +import logging import netaddr import re import socket @@ -23,87 +24,162 @@ from compass.utils import util def is_valid_ip(name, ip_addr, **kwargs): """Valid the format of an IP address.""" + if isinstance(ip_addr, list): + return all([ + is_valid_ip(name, item, **kwargs) for item in ip_addr + ]) try: netaddr.IPAddress(ip_addr) except Exception: + logging.debug('%s invalid ip addr %s', name, ip_addr) return False return True def is_valid_network(name, ip_network, **kwargs): """Valid the format of an Ip network.""" + if isinstance(ip_network, list): + return all([ + is_valid_network(name, item, **kwargs) for item in ip_network + ]) try: netaddr.IPNetwork(ip_network) except Exception: + logging.debug('%s invalid network %s', name, ip_network) return False - return False + return True def is_valid_netmask(name, ip_addr, **kwargs): """Valid the format of a netmask.""" + if isinstance(ip_addr, list): + return all([ + is_valid_netmask(name, item, **kwargs) for item in ip_addr + ]) if not is_valid_ip(ip_addr): return False ip = netaddr.IPAddress(ip_addr) if ip.is_netmask(): return True - else: - return False + logging.debug('%s invalid netmask %s', name, ip_addr) + return False def is_valid_gateway(name, ip_addr, **kwargs): """Valid the format of gateway.""" + if isinstance(ip_addr, list): + return all([ + is_valid_gateway(name, item, **kwargs) for item in ip_addr + ]) if not is_valid_ip(ip_addr): return False ip = netaddr.IPAddress(ip_addr) if ip.is_private() or ip.is_public(): return True - else: - return False + logging.debug('%s invalid gateway %s', name, ip_addr) + return False def is_valid_dns(name, dns, **kwargs): """Valid the format of DNS.""" + if isinstance(dns, list): + return all([is_valid_dns(name, item, **kwargs) for item in dns]) if is_valid_ip(dns): return True try: socket.gethostbyname_ex(dns) except Exception: + logging.debug('%s invalid dns name %s', name, dns) return False return True +def is_valid_url(name, url, **kwargs): + """Valid the format of url.""" + if isinstance(url, list): + return all([ + is_valid_url(name, item, **kwargs) for item in url + ]) + if re.match( + r'^(http|https|ftp)://([0-9A-Za-z_-]+)(\.[0-9a-zA-Z_-]+)*' + r'(:\d+)?(/[0-9a-zA-Z_-]+)*$', + url + ): + return True + logging.debug( + '%s invalid url %s', name, url + ) + return False + + +def is_valid_domain(name, domain, **kwargs): + """Validate the format of domain.""" + if isinstance(domain, list): + return all([ + is_valid_domain(name, item, **kwargs) for item in domain + ]) + if re.match( + r'^([0-9a-zA-Z_-]+)(\.[0-9a-zA-Z_-]+)*$', + domain + ): + return True + logging.debug( + '%s invalid domain %s', name, domain + ) + return False + + def is_valid_username(name, username, **kwargs): """Valid the format of username.""" - return bool(username) + if bool(username): + return True + logging.debug( + '%s username is empty', name + ) def is_valid_password(name, password, **kwargs): """Valid the format of password.""" - return bool(password) + if bool(password): + return True + logging.debug('%s password is empty', name) + return False def is_valid_partition(name, partition, **kwargs): """Valid the format of partition name.""" if name != 'swap' and not name.startswith('/'): + logging.debug( + '%s is not started with / or swap', name + ) return False if 'size' not in partition and 'percentage' not in partition: + logging.debug( + '%s partition does not contain sie or percentage', + name + ) return False return True def is_valid_percentage(name, percentage, **kwargs): """Valid the percentage.""" - return 0 <= percentage <= 100 + if 0 <= percentage <= 100: + return True + logging.debug('%s invalid percentage %s', name, percentage) def is_valid_port(name, port, **kwargs): """Valid the format of port.""" - return 0 < port < 65536 + if 0 < port < 65536: + return True + logging.debug('%s invalid port %s', name, port) def is_valid_size(name, size, **kwargs): - if re.match(r'(\d+)(K|M|G|T)?', size): + if re.match(r'^(\d+)(K|M|G|T)$', size): return True + logging.debug('%s invalid size %s', name, size) return False diff --git a/compass/tasks/tasks.py b/compass/tasks/tasks.py index eec1204d..63fbd035 100644 --- a/compass/tasks/tasks.py +++ b/compass/tasks/tasks.py @@ -21,6 +21,7 @@ import logging from celery.signals import celeryd_init from celery.signals import setup_logging +from compass.actions import clean from compass.actions import delete from compass.actions import deploy from compass.actions import poll_switch @@ -168,6 +169,32 @@ def delete_host(deleter_email, host_id, cluster_ids): logging.exception(error) +@celery.task(name='compass.tasks.clean_os_installer') +def clean_os_installer( + os_installer_name, os_installer_settings +): + """Clean os installer.""" + try: + clean.clean_os_installer( + os_installer_name, os_installer_settings + ) + except Exception as error: + logging.excception(error) + + +@celery.task(name='compass.tasks.clean_package_installer') +def clean_package_installer( + package_installer_name, package_installer_settings +): + """Clean package installer.""" + try: + clean.clean_package_installer( + package_installer_name, package_installer_settings + ) + except Exception as error: + logging.excception(error) + + @celery.task(name='compass.tasks.poweron_host') def poweron_host(host_id): """Deploy the given cluster. diff --git a/conf/os_field/domain.conf b/conf/os_field/domain.conf new file mode 100644 index 00000000..fdf74289 --- /dev/null +++ b/conf/os_field/domain.conf @@ -0,0 +1,2 @@ +NAME = 'domain' +VALIDATOR = is_valid_domain diff --git a/conf/os_field/ip_list.conf b/conf/os_field/ip_list.conf new file mode 100644 index 00000000..28969cb2 --- /dev/null +++ b/conf/os_field/ip_list.conf @@ -0,0 +1,3 @@ +NAME = 'ip_list' +FIELD_TYPE = list +VALIDATOR = is_valid_ip diff --git a/conf/os_field/url.conf b/conf/os_field/url.conf new file mode 100644 index 00000000..b6a74f35 --- /dev/null +++ b/conf/os_field/url.conf @@ -0,0 +1,2 @@ +NAME = 'url' +VALIDATOR = is_valid_url diff --git a/conf/os_metadata/general.conf b/conf/os_metadata/general.conf index 8caf1571..b6a6c481 100644 --- a/conf/os_metadata/general.conf +++ b/conf/os_metadata/general.conf @@ -29,7 +29,7 @@ METADATA = { }, 'http_proxy': { '_self': { - 'field': 'general', + 'field': 'url', 'default_callback': default_proxy, 'options_callback': proxy_options, 'mapping_to': 'http_proxy' @@ -37,7 +37,7 @@ METADATA = { }, 'https_proxy': { '_self': { - 'field': 'general', + 'field': 'url', 'default_callback': default_proxy, 'options_callback': proxy_options, 'mapping_to': 'https_proxy' @@ -55,7 +55,7 @@ METADATA = { 'ntp_server': { '_self': { 'is_required': True, - 'field': 'general', + 'field': 'ip', 'default_callback': default_ntp_server, 'options_callback': ntp_server_options, 'mapping_to': 'ntp_server' @@ -64,7 +64,7 @@ METADATA = { 'dns_servers': { '_self': { 'is_required': True, - 'field': 'general_list', + 'field': 'ip_list', 'default_callback': default_dns_servers, 'options_callback': dns_servers_options, 'mapping_to': 'nameservers' @@ -72,7 +72,7 @@ METADATA = { }, 'domain': { '_self': { - 'field': 'general', + 'field': 'domain', 'is_required' : True, 'default_callback': default_domain, 'options_callback': domain_options, @@ -96,7 +96,7 @@ METADATA = { }, 'local_repo': { '_self': { - 'field': 'general', + 'field': 'url', 'default_callback': default_localrepo, 'mapping_to': 'local_repo' } diff --git a/install/install_func.sh b/install/install_func.sh index 1f4df8e9..40670c1e 100755 --- a/install/install_func.sh +++ b/install/install_func.sh @@ -41,21 +41,18 @@ copy2dir() git clean -x -f git checkout $git_branch git reset --hard remotes/origin/$git_branch + git clean -x -f -d -q cd - else echo "create $destdir" mkdir -p $destdir - git clone $repo $destdir + git clone $repo $destdir -b $git_branch if [ $? -ne 0 ]; then echo "failed to git clone $repo $destdir" exit 1 else echo "git clone $repo $destdir suceeded" fi - cd $destdir - git checkout $git_branch - git reset --hard remotes/origin/$git_branch - cd - fi cd $destdir if [[ -z $ZUUL_PROJECT ]]; then diff --git a/install/prepare.sh b/install/prepare.sh index f0c41794..c5adf949 100755 --- a/install/prepare.sh +++ b/install/prepare.sh @@ -125,6 +125,12 @@ else fi cd $SCRIPT_DIR +remote_branch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u}) +if [[ "$?" != "0" ]]; then + remote_branch="origin/master" +fi +local_branch=$(echo ${remote_branch} | sed -e 's/origin\///g') + if [ -z $WEB_SOURCE ]; then echo "web source $WEB_SOURCE is not set" exit 1 @@ -135,7 +141,7 @@ if [ -z $ADAPTERS_SOURCE ]; then echo "adpaters source $ADAPTERS_SOURCE is not set" exit 1 fi -copy2dir "$ADAPTERS_SOURCE" "$ADAPTERS_HOME" "stackforge/compass-adapters" dev/experimental || exit $? +copy2dir "$ADAPTERS_SOURCE" "$ADAPTERS_HOME" "stackforge/compass-adapters" ${local_branch} || exit $? if [ "$tempest" == "true" ]; then echo "download tempest packages" diff --git a/regtest/regtest.sh b/regtest/regtest.sh index 2aa9498b..bc93535b 100755 --- a/regtest/regtest.sh +++ b/regtest/regtest.sh @@ -79,8 +79,6 @@ for i in `seq $VIRT_NUM`; do exit 1 fi - echo "make pxe${i} reboot if installation failing." - sed -i "// a\ " /etc/libvirt/qemu/pxe${i}.xml echo "check pxe${i} state" state=$(virsh domstate pxe${i}) if [[ "$state" == "running" ]]; then @@ -92,6 +90,10 @@ for i in `seq $VIRT_NUM`; do fi fi + echo "make pxe${i} reboot if installation failing." + sed -i "// a\ " /etc/libvirt/qemu/pxe${i}.xml + virsh define /etc/libvirt/qemu/pxe${i}.xml + echo "start pxe${i}" virsh start pxe${i} if [[ "$?" != "0" ]]; then diff --git a/tox.ini b/tox.ini index d05e7b02..5b714197 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ commands = python setup.py testr --coverage --testr-args='{posargs}' downloadcache = ~/cache/pip [flake8] -ignore = H302,H233,H803,F401 +ignore = H302,H304,H233,H803,F401 show-source = true builtins = _ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build