murano-conductor/muranoconductor/metadata.py

# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import tarfile
import shutil
import tempfile
import hashlib
from glob import glob
from metadataclient.common.exceptions import CommunicationError
from muranoconductor import config
from metadataclient.v1.client import Client
import os
from keystoneclient.v2_0 import client as ksclient
from keystoneclient.exceptions import EndpointNotFound

from openstack.common import log as logging

CHUNK_SIZE = 1 << 20  # 1MB

log = logging.getLogger(__name__)
CONF = config.CONF


class MetadataException(BaseException):
    # Inherited not from Exception in purpose:
    # On this exception ack message would not be sent
    pass


def _unpack_data_archive(task_id, hash):
    archive_name = hash + '.tar.gz'
    if not tarfile.is_tarfile(archive_name):
        raise MetadataException('Received invalid file {0} from Metadata '
                                'Repository'.format(hash))
    dst_dir = task_id
    if not os.path.exists(dst_dir):
        os.mkdir(dst_dir)
    tar = tarfile.open(archive_name, 'r:gz')
    try:
        tar.extractall(path=dst_dir)
    finally:
        tar.close()
    return dst_dir


def get_endpoint(token_id, tenant_id):
    endpoint = CONF.murano_metadata_url
    if not endpoint:
        keystone_settings = CONF.keystone

        client = ksclient.Client(auth_url=keystone_settings.auth_url,
                                 token=token_id)

        client.authenticate(
            auth_url=keystone_settings.auth_url,
            tenant_id=tenant_id,
            token=token_id)

        try:
            endpoint = client.service_catalog.url_for(
                service_type='murano-metadata')
        except EndpointNotFound:
            endpoint = 'http://localhost:8084/v1'
            log.warning(
                'Murano Metadata API location could not be found in the '
                'Keystone Service Catalog, using default: {0}'.format(
                    endpoint))
    return endpoint


def metadataclient(token_id, tenant_id):
    endpoint = get_endpoint(token_id, tenant_id)
    return Client(endpoint=endpoint, token=token_id)


def get_metadata(task_id, token_id, tenant_id):
    hash = _check_existing_hash()
    try:
        log.debug('Retrieving metadata from Murano Metadata Repository')
        resp, body_iter = metadataclient(token_id, tenant_id).\
            metadata_client.get_conductor_data(hash)
    except CommunicationError as e:
        if hash:
            log.warning('Metadata update failed: '
                        'Unable to connect Metadata Repository due to {0}. '
                        'Using existing version of metadata'.format(e))
        else:
            log.exception(e)
            exc_type, exc_value, exc_traceback = sys.exc_info()
            raise MetadataException('Unable to get data '
                                    'from Metadata Repository due to {0}: '
                                    '{1}'.format(exc_type.__name__, exc_value))

    else:
        if resp.status == 304:
            log.debug('Metadata unmodified. Using existing archive.')

        elif resp.status == 200:
            with tempfile.NamedTemporaryFile(delete=False) as archive:
                for chunk in body_iter:
                    archive.write(chunk)
            hash = _get_hash(archive.name)
            shutil.move(archive.name, hash + '.tar.gz')
        else:
            msg = 'Metadata update failed: '    \
                  'Got {0} status in response.'.format(resp.status)
            if hash:
                log.warning(msg + ' Using existing version of metadata.')
            else:
                raise MetadataException(msg)
    return _unpack_data_archive(task_id, hash)


def release(folder):
    log.debug('Deleting metadata folder {0}'.format(folder))
    try:
        shutil.rmtree(folder)
    except Exception as e:
        log.exception('Unable to delete folder {0} with '
                      'task metadata due to {1}'.format(folder, e))


def prepare(data_dir):
    if not os.path.exists(data_dir):
        os.makedirs(data_dir)
        log.info("Creating directory '{0}' to store "
                 "conductor data".format(data_dir))
    init_scripts_dst = os.path.join(data_dir,
                                    os.path.basename(CONF.init_scripts_dir))
    if os.path.exists(init_scripts_dst):
        log.info("Found existing init scripts directory at"
                 " '{0}'. Deleting it.'".format(init_scripts_dst))
        shutil.rmtree(init_scripts_dst)
    log.info("Copying init scripts directory from '{0}' "
             "to '{1}'".format(CONF.init_scripts_dir, init_scripts_dst))
    shutil.copytree(CONF.init_scripts_dir, init_scripts_dst)

    agent_config_dst = os.path.join(data_dir,
                                    os.path.basename(CONF.agent_config_dir))
    if os.path.exists(agent_config_dst):
        log.info("Found existing agent config directory at"
                 " '{0}'. Deleting it.'".format(agent_config_dst))
        shutil.rmtree(agent_config_dst)
    log.info("Copying agent config directory from '{0}' "
             "to '{1}'".format(CONF.agent_config_dir, agent_config_dst))
    shutil.copytree(CONF.agent_config_dir, agent_config_dst)
    os.chdir(data_dir)


def _get_hash(archive_path):
    """Calculate SHA1-hash of archive file.

    SHA-1 take a bit more time than MD5 (see http://tinyurl.com/kpj5jy7),
    but is more secure.
    """
    if os.path.exists(archive_path):
        sha1 = hashlib.sha1()
        with open(archive_path) as f:
            buf = f.read(CHUNK_SIZE)
            while buf:
                sha1.update(buf)
                buf = f.read(CHUNK_SIZE)
        hsum = sha1.hexdigest()
        log.debug("Archive '{0}' has hash-sum {1}".format(archive_path, hsum))
        return hsum
    else:
        log.info("Archive '{0}' doesn't exist, no hash to calculate".format(
            archive_path))
        return None


def _check_existing_hash():
    hash_archives = glob('*.tar.gz')
    if not hash_archives:
        hash = None
    else:
        if len(hash_archives) > 1:
            log.warning('There are to metadata archive. Deleting them both')
            for item in hash_archives:
                os.remove(item)
                hash = None
        else:
            file_name, extension = hash_archives[0].split('.', 1)
            hash = file_name
    return hash