clean installers when refresh

Change-Id: I86c27b0d9be0af6ef58748d97938a2df54b297c3

bin/clean_installers.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Scripts to delete cluster and it hosts"""
+import logging
+import os
+import os.path
+import sys
+
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(current_dir)
+
+
+import switch_virtualenv
+
+from compass.actions import clean
+from compass.db.api import adapter_holder as adapter_api
+from compass.db.api import database
+from compass.db.api import user as user_api
+from compass.tasks.client import celery
+from compass.utils import flags
+from compass.utils import logsetting
+from compass.utils import setting_wrapper as setting
+
+
+flags.add_bool('async',
+               help='run in async mode',
+               default=True)
+
+flags.add('os_installers',
+          help='comma seperated os installers',
+          default='')
+flags.add('package_installers',
+          help='comma separated package installers',
+          default='')
+
+
+def clean_installers():
+    os_installers = [
+        os_installer
+        for os_installer in flags.OPTIONS.os_installers.split(',')
+        if os_installer
+    ]
+    package_installers = [
+        package_installer
+        for package_installer in flags.OPTIONS.package_installers.split(',')
+        if package_installer
+    ]
+    user = user_api.get_user_object(setting.COMPASS_ADMIN_EMAIL)
+    adapters = adapter_api.list_adapters(user)
+    filtered_os_installers = {}
+    filtered_package_installers = {}
+    for adapter in adapters:
+        logging.info(
+            'got adapter: %s', adapter
+        )
+        if 'os_installer' in adapter:
+            os_installer = adapter['os_installer']
+            os_installer_name = os_installer['alias']
+            if not os_installers or os_installer_name in os_installers:
+                filtered_os_installers[os_installer_name] = os_installer
+            else:
+                logging.info(
+                    'ignore os isntaller %s', os_installer_name
+                )
+        else:
+            logging.info(
+                'cannot find os installer in adapter %s',
+                adapter['name']
+            )
+        if 'package_installer' in adapter:
+            package_installer = adapter['package_installer']
+            package_installer_name = package_installer['alias']
+            if (
+                not package_installers or
+                package_installer_name in package_installers
+            ):
+                filtered_package_installers[package_installer_name] = (
+                    package_installer
+                )
+            else:
+                logging.info(
+                    'ignore package installer %s', package_installer_name
+                )
+        else:
+            logging.info(
+                'cannot find package installer in adapter %s',
+                adapter['name']
+            )
+    logging.info(
+        'clean os installers: %s', filtered_os_installers.keys()
+    )
+    logging.info(
+        'clean package installers: %s', filtered_package_installers.keys()
+    )
+    if flags.OPTIONS.async:
+        for os_installer_name, os_installer in filtered_os_installers.items():
+            celery.send_task(
+                'compass.tasks.clean_os_installer',
+                (
+                    os_installer['name'],
+                    os_installer['settings']
+                )
+            )
+        for package_installer_name, package_installer in (
+            filtered_package_installers.items()
+        ):
+            celery.send_task(
+                'compass.tasks.clean_package_installer',
+                (
+                    package_installer['name'],
+                    package_installer['settings']
+                )
+            )
+    else:
+        for os_installer_name, os_installer in (
+            filtered_os_installers.items()
+        ):
+            try:
+                clean.clean_os_installer(
+                    os_installer['name'],
+                    os_installer['settings']
+                )
+            except Exception as error:
+                logging.error(
+                    'failed to clean os installer %s', os_installer_name
+                )
+                logging.exception(error)
+        for package_installer_name, package_installer in (
+            filtered_package_installers.items()
+        ):
+            try:
+                clean.clean_package_installer(
+                    package_installer['name'],
+                    package_installer['settings']
+                )
+            except Exception as error:
+                logging.error(
+                    'failed to clean package installer %s',
+                    package_installer_name
+                )
+                logging.exception(error)
+
+
+if __name__ == '__main__':
+    flags.init()
+    logsetting.init()
+    database.init()
+    clean_installers()

bin/delete_clusters.py

@@ -56,8 +56,11 @@ def delete_clusters():
         if clustername
     ]
     user = user_api.get_user_object(setting.COMPASS_ADMIN_EMAIL)
+    list_cluster_args = {}
+    if clusternames:
+        list_cluster_args['name'] = clusternames
     clusters = cluster_api.list_clusters(
-        user, name=clusternames
+        user, **list_cluster_args
     )
     delete_underlying_host = flags.OPTIONS.delete_hosts
     for cluster in clusters:

bin/refresh.sh

@@ -2,10 +2,7 @@
 set -e
 service mysqld restart
 /opt/compass/bin/manage_db.py createdb
-echo "You may run '/opt/compass/bin/clean_nodes.sh' to clean nodes on chef server"
+/opt/compass/bin/clean_installers.py
-echo "You may run '/opt/compass/bin/clean_clients.sh' to clean clients on chef server"
-echo "you may run '/opt/compass/bin/clean_environments.sh' to clean environments on chef server"
-echo "you may run '/opt/compass/bin/remove_systems.sh' to clean systems on cobbler"
 /opt/compass/bin/clean_installation_logs.py
 service httpd restart
 service rsyslog restart

compass/actions/clean.py

@@ -0,0 +1,182 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to clean installers
+"""
+import chef
+import logging
+import xmlrpclib
+
+from compass.actions import util
+
+
+class CobblerInstaller(object):
+    """cobbler installer"""
+    CREDENTIALS = "credentials"
+    USERNAME = 'username'
+    PASSWORD = 'password'
+
+    INSTALLER_URL = "cobbler_url"
+
+    def __init__(self, settings):
+        username = settings[self.CREDENTIALS][self.USERNAME]
+        password = settings[self.CREDENTIALS][self.PASSWORD]
+        cobbler_url = settings[self.INSTALLER_URL]
+        try:
+            self.remote = xmlrpclib.Server(cobbler_url)
+            self.token = self.remote.login(username, password)
+            logging.info('cobbler %s client created', cobbler_url)
+        except Exception as error:
+            logging.error(
+                'failed to login %s with (%s, %s)',
+                cobbler_url, username, password
+            )
+            logging.exception(error)
+
+    def clean(self):
+        systems = self.remote.get_systems()
+        for system in systems:
+            system_name = system['name']
+            try:
+                self.remote.remove_system(system_name, self.token)
+                logging.info('system %s is removed', system_name)
+            except Exception as error:
+                logging.error(
+                    'failed to remove system %s', system_name
+                )
+                logging.exception(error)
+
+
+class ChefInstaller(object):
+    DATABAGS = "databags"
+    CHEFSERVER_URL = "chef_url"
+    CHEFSERVER_DNS = "chef_server_dns"
+    CHEFSERVER_IP = "chef_server_ip"
+    KEY_DIR = "key_dir"
+    CLIENT = "client_name"
+
+    def __init__(self, settings):
+        installer_url = settings.get(self.CHEFSERVER_URL, None)
+        key_dir = settings.get(self.KEY_DIR, None)
+        client = settings.get(self.CLIENT, None)
+        try:
+            if installer_url and key_dir and client:
+                self.api = chef.ChefAPI(installer_url, key_dir, client)
+            else:
+                self.api = chef.autoconfigure()
+            logging.info(
+                'chef client created %s(%s, %s)',
+                installer_url, key_dir, client
+            )
+        except Exception as error:
+            logging.error(
+                'failed to create chef client %s(%s, %s)',
+                installer_url, key_dir, client
+            )
+            logging.exception(error)
+
+    def clean(self):
+        try:
+            for node_name in chef.Node.list(api=self.api):
+                node = chef.Node(node_name, api=self.api)
+                node.delete()
+                logging.info('delete node %s', node_name)
+        except Exception as error:
+            logging.error('failed to delete some nodes')
+            logging.exception(error)
+
+        try:
+            for client_name in chef.Client.list(api=self.api):
+                if client_name in ['chef-webui', 'chef-validator']:
+                    continue
+                client = chef.Client(client_name, api=self.api)
+                client.delete()
+                logging.info('delete client %s', client_name)
+        except Exception as error:
+            logging.error('failed to delete some clients')
+            logging.exception(error)
+
+        try:
+            for env_name in chef.Environment.list(api=self.api):
+                if env_name == '_default':
+                    continue
+                env = chef.Environment(env_name, api=self.api)
+                env.delete()
+                logging.info('delete env %s', env_name)
+        except Exception as error:
+            logging.error('failed to delete some envs')
+            logging.exception(error)
+
+        try:
+            for databag_name in chef.DataBag.list(api=self.api):
+                databag = chef.DataBag(databag_name, api=self.api)
+                for item_name, item in databag.items():
+                    item.delete()
+                    logging.info(
+                        'delete item %s from databag %s',
+                        item_name, databag_name
+                    )
+        except Exception as error:
+            logging.error('failed to delete some databag items')
+            logging.exception(error)
+
+
+OS_INSTALLERS = {
+    'cobbler': CobblerInstaller
+}
+PK_INSTALLERS = {
+    'chef_installer': ChefInstaller
+}
+
+
+def clean_os_installer(
+    os_installer_name, os_installer_settings
+):
+    with util.lock('serialized_action', timeout=100) as lock:
+        if not lock:
+            raise Exception(
+                'failed to acquire lock to clean os installer'
+            )
+
+        if os_installer_name not in OS_INSTALLERS:
+            logging.error(
+                '%s not found in os_installers',
+                os_installer_name
+            )
+
+        os_installer = OS_INSTALLERS[os_installer_name](
+            os_installer_settings
+        )
+        os_installer.clean()
+
+
+def clean_package_installer(
+    package_installer_name, package_installer_settings
+):
+    with util.lock('serialized_action', timeout=100) as lock:
+        if not lock:
+            raise Exception(
+                'failed to acquire lock to clean package installer'
+            )
+
+        if package_installer_name not in PK_INSTALLERS:
+            logging.error(
+                '%s not found in os_installers',
+                package_installer_name
+            )
+
+        package_installer = PK_INSTALLERS[package_installer_name](
+            package_installer_settings
+        )
+        package_installer.clean()

compass/db/api/metadata.py

@@ -301,17 +301,28 @@ def _validate_self(
     metadata, whole_check,
     **kwargs
 ):
+    logging.debug('validate config self %s', config_path)
     if '_self' not in metadata:
         if isinstance(config, dict):
             _validate_config(
                 config_path, config, metadata, whole_check, **kwargs
             )
         return
-    field_type = metadata['_self'].get('field_type', 'basestring')
+    field_type = metadata['_self'].get('field_type', basestring)
     if not isinstance(config, field_type):
         raise exception.InvalidParameter(
             '%s config type is not %s' % (config_path, field_type)
         )
+    is_required = metadata['_self'].get(
+        'is_required', False
+    )
+    required_in_whole_config = metadata['_self'].get(
+        'required_in_whole_config', False
+    )
+    if isinstance(config, basestring):
+        if config == '' and not is_required and not required_in_whole_config:
+            # ignore empty config when it is optional
+            return
     required_in_options = metadata['_self'].get(
         'required_in_options', False
     )
@@ -333,6 +344,7 @@ def _validate_self(
                     '%s config is not in %s' % (config_path, options)
                 )
     validator = metadata['_self'].get('validator', None)
+    logging.debug('validate by validator %s', validator)
     if validator:
         if not validator(config_key, config, **kwargs):
             raise exception.InvalidParameter(
@@ -348,6 +360,7 @@ def _validate_config(
     config_path, config, metadata, whole_check,
     **kwargs
 ):
+    logging.debug('validate config %s', config_path)
     generals = {}
     specified = {}
     for key, value in metadata.items():

compass/db/validator.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 """Validator methods."""
+import logging
 import netaddr
 import re
 import socket
@@ -23,87 +24,162 @@ from compass.utils import util
 
 def is_valid_ip(name, ip_addr, **kwargs):
     """Valid the format of an IP address."""
+    if isinstance(ip_addr, list):
+        return all([
+            is_valid_ip(name, item, **kwargs) for item in ip_addr
+        ])
     try:
         netaddr.IPAddress(ip_addr)
     except Exception:
+        logging.debug('%s invalid ip addr %s', name, ip_addr)
         return False
     return True
 
 
 def is_valid_network(name, ip_network, **kwargs):
     """Valid the format of an Ip network."""
+    if isinstance(ip_network, list):
+        return all([
+            is_valid_network(name, item, **kwargs) for item in ip_network
+        ])
     try:
         netaddr.IPNetwork(ip_network)
     except Exception:
+        logging.debug('%s invalid network %s', name, ip_network)
         return False
-    return False
+    return True
 
 
 def is_valid_netmask(name, ip_addr, **kwargs):
     """Valid the format of a netmask."""
+    if isinstance(ip_addr, list):
+        return all([
+            is_valid_netmask(name, item, **kwargs) for item in ip_addr
+        ])
     if not is_valid_ip(ip_addr):
         return False
     ip = netaddr.IPAddress(ip_addr)
     if ip.is_netmask():
         return True
-    else:
+    logging.debug('%s invalid netmask %s', name, ip_addr)
-        return False
+    return False
 
 
 def is_valid_gateway(name, ip_addr, **kwargs):
     """Valid the format of gateway."""
+    if isinstance(ip_addr, list):
+        return all([
+            is_valid_gateway(name, item, **kwargs) for item in ip_addr
+        ])
     if not is_valid_ip(ip_addr):
         return False
     ip = netaddr.IPAddress(ip_addr)
     if ip.is_private() or ip.is_public():
         return True
-    else:
+    logging.debug('%s invalid gateway %s', name, ip_addr)
-        return False
+    return False
 
 
 def is_valid_dns(name, dns, **kwargs):
     """Valid the format of DNS."""
+    if isinstance(dns, list):
+        return all([is_valid_dns(name, item, **kwargs) for item in dns])
     if is_valid_ip(dns):
         return True
     try:
         socket.gethostbyname_ex(dns)
     except Exception:
+        logging.debug('%s invalid dns name %s', name, dns)
         return False
     return True
 
 
+def is_valid_url(name, url, **kwargs):
+    """Valid the format of url."""
+    if isinstance(url, list):
+        return all([
+            is_valid_url(name, item, **kwargs) for item in url
+        ])
+    if re.match(
+        r'^(http|https|ftp)://([0-9A-Za-z_-]+)(\.[0-9a-zA-Z_-]+)*'
+        r'(:\d+)?(/[0-9a-zA-Z_-]+)*$',
+        url
+    ):
+        return True
+    logging.debug(
+        '%s invalid url %s', name, url
+    )
+    return False
+
+
+def is_valid_domain(name, domain, **kwargs):
+    """Validate the format of domain."""
+    if isinstance(domain, list):
+        return all([
+            is_valid_domain(name, item, **kwargs) for item in domain
+        ])
+    if re.match(
+        r'^([0-9a-zA-Z_-]+)(\.[0-9a-zA-Z_-]+)*$',
+        domain
+    ):
+        return True
+    logging.debug(
+        '%s invalid domain %s', name, domain
+    )
+    return False
+
+
 def is_valid_username(name, username, **kwargs):
     """Valid the format of username."""
-    return bool(username)
+    if bool(username):
+        return True
+    logging.debug(
+        '%s username is empty', name
+    )
 
 
 def is_valid_password(name, password, **kwargs):
     """Valid the format of password."""
-    return bool(password)
+    if bool(password):
+        return True
+    logging.debug('%s password is empty', name)
+    return False
 
 
 def is_valid_partition(name, partition, **kwargs):
     """Valid the format of partition name."""
     if name != 'swap' and not name.startswith('/'):
+        logging.debug(
+            '%s is not started with / or swap', name
+        )
         return False
     if 'size' not in partition and 'percentage' not in partition:
+        logging.debug(
+            '%s partition does not contain sie or percentage',
+            name
+        )
         return False
     return True
 
 
 def is_valid_percentage(name, percentage, **kwargs):
     """Valid the percentage."""
-    return 0 <= percentage <= 100
+    if 0 <= percentage <= 100:
+        return True
+    logging.debug('%s invalid percentage %s', name, percentage)
 
 
 def is_valid_port(name, port, **kwargs):
     """Valid the format of port."""
-    return 0 < port < 65536
+    if 0 < port < 65536:
+        return True
+    logging.debug('%s invalid port %s', name, port)
 
 
 def is_valid_size(name, size, **kwargs):
-    if re.match(r'(\d+)(K|M|G|T)?', size):
+    if re.match(r'^(\d+)(K|M|G|T)$', size):
         return True
+    logging.debug('%s invalid size %s', name, size)
     return False
 
 

compass/tasks/tasks.py

@@ -21,6 +21,7 @@ import logging
 from celery.signals import celeryd_init
 from celery.signals import setup_logging
 
+from compass.actions import clean
 from compass.actions import delete
 from compass.actions import deploy
 from compass.actions import poll_switch
@@ -168,6 +169,32 @@ def delete_host(deleter_email, host_id, cluster_ids):
         logging.exception(error)
 
 
+@celery.task(name='compass.tasks.clean_os_installer')
+def clean_os_installer(
+    os_installer_name, os_installer_settings
+):
+    """Clean os installer."""
+    try:
+        clean.clean_os_installer(
+            os_installer_name, os_installer_settings
+        )
+    except Exception as error:
+        logging.excception(error)
+
+
+@celery.task(name='compass.tasks.clean_package_installer')
+def clean_package_installer(
+    package_installer_name, package_installer_settings
+):
+    """Clean package installer."""
+    try:
+        clean.clean_package_installer(
+            package_installer_name, package_installer_settings
+        )
+    except Exception as error:
+        logging.excception(error)
+
+
 @celery.task(name='compass.tasks.poweron_host')
 def poweron_host(host_id):
     """Deploy the given cluster.

conf/os_field/domain.conf

@@ -0,0 +1,2 @@
+NAME = 'domain'
+VALIDATOR = is_valid_domain

conf/os_field/ip_list.conf

@@ -0,0 +1,3 @@
+NAME = 'ip_list'
+FIELD_TYPE = list
+VALIDATOR = is_valid_ip

conf/os_field/url.conf

@@ -0,0 +1,2 @@
+NAME = 'url'
+VALIDATOR = is_valid_url

conf/os_metadata/general.conf

@@ -29,7 +29,7 @@ METADATA = {
         },
         'http_proxy': {
             '_self': {
-                'field': 'general',
+                'field': 'url',
                 'default_callback': default_proxy,
                 'options_callback': proxy_options,
                 'mapping_to': 'http_proxy'
@@ -37,7 +37,7 @@ METADATA = {
         },
         'https_proxy': {
             '_self': {
-                'field': 'general',
+                'field': 'url',
                 'default_callback': default_proxy,
                 'options_callback': proxy_options,
                 'mapping_to': 'https_proxy'
@@ -55,7 +55,7 @@ METADATA = {
         'ntp_server': {
             '_self': {
                 'is_required': True,
-                'field': 'general',
+                'field': 'ip',
 		'default_callback': default_ntp_server,
                 'options_callback': ntp_server_options,
                 'mapping_to': 'ntp_server'
@@ -64,7 +64,7 @@ METADATA = {
         'dns_servers': {
             '_self': {
                 'is_required': True,
-                'field': 'general_list',
+                'field': 'ip_list',
 		'default_callback': default_dns_servers,
                 'options_callback': dns_servers_options,
                 'mapping_to': 'nameservers'
@@ -72,7 +72,7 @@ METADATA = {
         },
         'domain': {
             '_self': {
-                'field': 'general',
+                'field': 'domain',
                 'is_required' : True,
 		'default_callback': default_domain,
                 'options_callback': domain_options,
@@ -96,7 +96,7 @@ METADATA = {
         },
         'local_repo': {
             '_self': {
-                'field': 'general',
+                'field': 'url',
                 'default_callback': default_localrepo,
                 'mapping_to': 'local_repo'
             }

install/install_func.sh

@@ -41,21 +41,18 @@ copy2dir()
             git clean -x -f
             git checkout $git_branch
             git reset --hard remotes/origin/$git_branch
+	    git clean -x -f -d -q
 	    cd -
         else
             echo "create $destdir"
             mkdir -p $destdir
-            git clone $repo $destdir
+            git clone $repo $destdir -b $git_branch
             if [ $? -ne 0 ]; then
                 echo "failed to git clone $repo $destdir"
                 exit 1
             else
                 echo "git clone $repo $destdir suceeded"
             fi
-            cd $destdir
-	    git checkout $git_branch
-            git reset --hard remotes/origin/$git_branch
-	    cd -
         fi
 	cd $destdir
         if [[ -z $ZUUL_PROJECT ]]; then

install/prepare.sh

@@ -125,6 +125,12 @@ else
 fi
 
 cd $SCRIPT_DIR
+remote_branch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u})
+if [[ "$?" != "0" ]]; then
+    remote_branch="origin/master"
+fi
+local_branch=$(echo ${remote_branch} | sed -e 's/origin\///g')
+
 if [ -z $WEB_SOURCE ]; then
     echo "web source $WEB_SOURCE is not set"
     exit 1
@@ -135,7 +141,7 @@ if [ -z $ADAPTERS_SOURCE ]; then
     echo "adpaters source $ADAPTERS_SOURCE is not set"
     exit 1
 fi
-copy2dir "$ADAPTERS_SOURCE" "$ADAPTERS_HOME" "stackforge/compass-adapters" dev/experimental || exit $?
+copy2dir "$ADAPTERS_SOURCE" "$ADAPTERS_HOME" "stackforge/compass-adapters" ${local_branch} || exit $?
 
 if [ "$tempest" == "true" ]; then
     echo "download tempest packages"

regtest/regtest.sh

@@ -79,8 +79,6 @@ for i in `seq $VIRT_NUM`; do
         exit 1
     fi
 
-    echo "make pxe${i} reboot if installation failing."
-    sed -i "/<boot dev='hd'\/>/ a\    <bios useserial='yes' rebootTimeout='0'\/>" /etc/libvirt/qemu/pxe${i}.xml
     echo "check pxe${i} state"
     state=$(virsh domstate pxe${i})
     if [[ "$state" == "running" ]]; then
@@ -92,6 +90,10 @@ for i in `seq $VIRT_NUM`; do
         fi
     fi
 
+    echo "make pxe${i} reboot if installation failing."
+    sed -i "/<boot dev='hd'\/>/ a\    <bios useserial='yes' rebootTimeout='0'\/>" /etc/libvirt/qemu/pxe${i}.xml
+    virsh define /etc/libvirt/qemu/pxe${i}.xml
+
     echo "start pxe${i}"
     virsh start pxe${i}
     if [[ "$?" != "0" ]]; then

tox.ini

@@ -31,7 +31,7 @@ commands = python setup.py testr --coverage --testr-args='{posargs}'
 downloadcache = ~/cache/pip
 
 [flake8]
-ignore = H302,H233,H803,F401
+ignore = H302,H304,H233,H803,F401
 show-source = true
 builtins = _
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build