Add per-tenant isolation

* Fix some exceptions
* use tmpfile to store uploaded archive
Implements blueprint per-tenant-isolation
Change-Id: I9a302823fb208cc73808e573cf4a9812a0c9cf79

etc/murano-repository.conf.sample

@@ -5,7 +5,7 @@ host =  0.0.0.0
 port = 8084
 
 # Directory for cache, OS temp directory is used by default
-#cache_dir =
+# data_dir = /tmp/muranorepository-data
 
 # Show more verbose log output (sets INFO log level output)
 verbose = True

muranorepository/api/utils.py

@@ -12,6 +12,7 @@ from werkzeug import secure_filename
 from muranorepository.utils.parser import ManifestParser
 from muranorepository.utils.parser import serialize
 from muranorepository.utils.archiver import Archiver
+from muranorepository.utils import utils
 from muranorepository.consts import DATA_TYPES, MANIFEST
 from muranorepository.consts import CLIENTS_DICT
 from muranorepository.consts import ARCHIVE_PKG_NAME
@@ -23,16 +24,26 @@ CONF = cfg.CONF
 
 def reset_cache():
     try:
-        shutil.rmtree(CONF.cache_dir, ignore_errors=True)
+        cache_dir = utils.get_cache_folder()
-        os.mkdir(CONF.cache_dir)
+        shutil.rmtree(cache_dir, ignore_errors=True)
+        os.mkdir(cache_dir)
     except:
+        log.exception(_('Error while cleaning cache'))
         return make_response('Unable to reset cache', 500)
 
 
+def compose_path(data_type, path=None):
+    tenant_dir = utils.get_tenant_folder()
+    utils.check_tenant_dir_existence(tenant_dir)
+    return os.path.join(tenant_dir,
+                        getattr(CONF, data_type),
+                        path or '')
+
+
 def get_archive(client, hash_sum):
     types = CLIENTS_DICT.get(client)
     archive_manager = Archiver()
-    cache_dir = os.path.join(CONF.cache_dir, client)
+    cache_dir = os.path.join(utils.get_cache_folder(), client)
     if not os.path.exists(cache_dir):
         os.makedirs(cache_dir)
         existing_hash = None
@@ -45,6 +56,7 @@ def get_archive(client, hash_sum):
 
     if archive_manager.hashes_match(cache_dir, existing_hash, hash_sum):
         return None
+
     manifests = ManifestParser().parse()
     return archive_manager.create(cache_dir, manifests, types)
 
@@ -106,13 +118,6 @@ def save_file(request, data_type, path=None, filename=None):
     return jsonify(result='success')
 
 
-def compose_path(data_type, path=None):
-    if path:
-        return os.path.join(CONF.manifests, getattr(CONF, data_type), path)
-    else:
-        return os.path.join(CONF.manifests, getattr(CONF, data_type))
-
-
 def check_data_type(data_type):
     if data_type not in DATA_TYPES:
         abort(404)
@@ -147,11 +152,10 @@ def check_service_name(service_name):
 def perform_deletion(files_for_deletion, manifest_for_deletion):
     def backup_data():
         backup_dir = os.path.join(
-            os.path.dirname(CONF.manifests),
+            utils.get_cache_folder(),
-            'Backup_{0}'.format(datetime.datetime.utcnow())
+            'Backup_{0}'.format(datetime.datetime.utcnow()))
-        )
+        log.debug('Creating service data backup to {0}'.format(backup_dir))
-        log.debug(_('Creating service data backup to {0}'.format(backup_dir)))
+        shutil.copytree(utils.get_tenant_folder(), backup_dir)
-        shutil.copytree(CONF.manifests, backup_dir)
         return backup_dir
 
     def release_backup(backup):
@@ -163,12 +167,12 @@ def perform_deletion(files_for_deletion, manifest_for_deletion):
 
     def restore_backup(backup):
         log.debug('Restore service data after unsuccessful deletion')
-        shutil.rmtree(CONF.manifests, ignore_errors=True)
+        shutil.rmtree(utils.get_tenant_folder(), ignore_errors=True)
-        os.rename(backup, CONF.manifests)
+        os.rename(backup, utils.get_tenant_folder())
 
     backup_dir = backup_data()
     service_name = manifest_for_deletion.full_service_name
-    path_to_manifest = os.path.join(CONF.manifests,
+    path_to_manifest = os.path.join(utils.get_tenant_folder(),
                                     '{0}-manifest.yaml'.format(service_name))
     try:
         if os.path.exists(path_to_manifest):
@@ -176,8 +180,8 @@ def perform_deletion(files_for_deletion, manifest_for_deletion):
             os.remove(path_to_manifest)
 
         for data_type, files in files_for_deletion.iteritems():
-            data_type_dir = os.path.join(CONF.manifests, getattr(CONF,
+            data_type_dir = os.path.join(utils.get_tenant_folder(),
-                                                                 data_type))
+                                         getattr(CONF, data_type))
             for file in files:
                 path_to_delete = os.path.join(data_type_dir, file)
                 if os.path.exists(path_to_delete):
@@ -205,11 +209,9 @@ def save_archive(request):
         path_to_archive = uploaded_file.name
     else:
         file_to_upload = request.files.get('file')
-        if file_to_upload:
+        if not file_to_upload:
-            filename = secure_filename(file_to_upload.filename)
-        else:
             return err_resp
-        path_to_archive = os.path.join(CONF.cache_dir, filename)
+        path_to_archive = tempfile.NamedTemporaryFile(delete=False).name
         file_to_upload.save(path_to_archive)
     return path_to_archive
 
@@ -230,7 +232,7 @@ def create_or_update_service(service_id, data):
         if not data.get(parameter):
             data[parameter] = optional[parameter]
 
-    path_to_manifest = os.path.join(CONF.manifests,
+    path_to_manifest = os.path.join(utils.get_tenant_folder(),
                                     service_id + '-manifest.yaml')
 
     backup_done = False

muranorepository/cmd/run.py

@@ -53,13 +53,14 @@ def main():
     log.setup('muranorepository')
 
     #configuring and initializing cache directory
-    if cfg.CONF.cache_dir is None:
+    if cfg.CONF.data_dir is None:
-        cfg.CONF.cache_dir = os.path.join(
+        cfg.CONF.data_dir = os.path.join(
-            tempfile.gettempdir(), 'muranorepository-cache'
+            tempfile.gettempdir(), 'muranorepository-data'
         )
-    if not os.path.exists(cfg.CONF.cache_dir):
+
-        os.mkdir(cfg.CONF.cache_dir)
+    if not os.path.exists(cfg.CONF.data_dir):
-    LOG.info('Cache is located at: {0}'.format(cfg.CONF.cache_dir))
+        os.mkdir(cfg.CONF.data_dir)
+    LOG.info('Cache is located at: {0}'.format(cfg.CONF.data_dir))
 
     app = server.make_app({
         'auth_host': cfg.CONF.keystone.auth_host,

muranorepository/config.py

@@ -21,7 +21,7 @@ server_opts = [
     cfg.IntOpt('port', default=5000)
 ]
 
-cache_opt = cfg.StrOpt('cache_dir')
+cache_opt = cfg.StrOpt('data_dir')
 
 keystone_opts = [
     cfg.StrOpt('auth_host', default='localhost'),

muranorepository/consts.py

@@ -30,3 +30,6 @@ ARCHIVE_PKG_NAME = 'data.tar.gz'
 UI_FIELDS_IN_MANIFEST = {'description': 'description',
                          'type': 'full_service_name',
                          'name': 'service_display_name'}
+
+SERVICE_DEFINITIONS_FOLDER_NAME = 'service_definitions'
+CACHE_FOLDER_NAME = 'cache'

muranorepository/utils/archiver.py

@@ -23,6 +23,7 @@ from oslo.config import cfg
 from .parser import serialize
 from muranorepository.consts import DATA_TYPES, ARCHIVE_PKG_NAME
 from muranorepository.consts import UI, UI_FIELDS_IN_MANIFEST
+from muranorepository.utils import utils
 from muranorepository.openstack.common.gettextutils import _  # noqa
 CONF = cfg.CONF
 
@@ -206,16 +207,17 @@ class Archiver(object):
 
                 if hasattr(manifest, data_type):
                     file_list = getattr(manifest, data_type)
-                    scr_directory = os.path.join(
+                    src_directory = os.path.join(
-                        CONF.manifests, self.src_directories[data_type])
+                        utils.get_tenant_folder(),
+                        self.src_directories[data_type])
                     dst_directory = os.path.join(
                         temp_dir, self.dst_directories[data_type])
                     if data_type == UI:
                         self._compose_ui_forms(manifest, file_list,
-                                               scr_directory, dst_directory)
+                                               src_directory, dst_directory)
                     else:
                         self._copy_data(file_list,
-                                        scr_directory,
+                                        src_directory,
                                         dst_directory)
                 else:
                     log.info(
@@ -234,23 +236,25 @@ class Archiver(object):
         for data_type in DATA_TYPES:
             if hasattr(manifest, data_type):
                 file_list = getattr(manifest, data_type)
-                scr_directory = os.path.join(
+                src_directory = os.path.join(
-                    CONF.manifests, self.src_directories[data_type])
+                    utils.get_tenant_folder(), self.src_directories[data_type])
                 dst_directory = os.path.join(
                     temp_dir, self.dst_directories[data_type])
-                self._copy_data(file_list, scr_directory, dst_directory)
+                self._copy_data(file_list, src_directory, dst_directory)
             else:
                 log.info(
                     _('{0} manifest has no file definitions for '
                       '{1}'.format(manifest.service_display_name, data_type)))
         #Add manifest file into archive
         manifest_filename = manifest.full_service_name + '-manifest.yaml'
-        self._copy_data([manifest_filename], CONF.manifests, temp_dir)
+        self._copy_data([manifest_filename],
+                        utils.get_tenant_folder(),
+                        temp_dir)
         return self._compose_archive(file_name, temp_dir)
 
     def remove_existing_hash(self, cache_dir, hash):
         path = os.path.join(cache_dir, hash)
-        log.info(_('Deleting archive package from {0}.'.format(path)))
+        log.info('Deleting archive package from {0}.'.format(path))
         shutil.rmtree(path, ignore_errors=True)
 
     def extract(self, path_to_archive):
@@ -260,6 +264,7 @@ class Archiver(object):
         return value - True if succeeded , False otherwise
         """
         try:
+            root_folder = utils.get_tenant_folder()
             path_to_extract = tempfile.mkdtemp()
             archive = tarfile.open(path_to_archive)
             try:
@@ -277,7 +282,7 @@ class Archiver(object):
                             'file in archive'))
                 return False
 
-            shutil.copy(manifests[0], CONF.manifests)
+            shutil.copy(manifests[0], root_folder)
             #Todo: Check manifest is valid
             for item in os.listdir(path_to_extract):
                 item_path = os.path.join(path_to_extract, item)
@@ -298,7 +303,7 @@ class Archiver(object):
                         self._copy_data(file_list,
                                         item_path,
                                         os.path.join(
-                                            CONF.manifests,
+                                            root_folder,
                                             self.src_directories[item]),
                                         overwrite=False)
                     else:

muranorepository/utils/parser.py

@@ -16,9 +16,11 @@ import os
 import yaml
 from oslo.config import cfg
 import logging as log
+
 from muranorepository.manifest import Manifest
 from muranorepository.consts import DATA_TYPES, MANIFEST
 from muranorepository.openstack.common.gettextutils import _  # noqa
+from muranorepository.utils import utils
 CONF = cfg.CONF
 
 
@@ -48,8 +50,9 @@ def serialize(data):
 class ManifestParser(object):
     def __init__(self, manifest_directory=None):
         if not manifest_directory:
-            manifest_directory = CONF.manifests
+            manifest_directory = utils.get_tenant_folder()
         self.manifest_directory = manifest_directory
+        utils.check_tenant_dir_existence(self.manifest_directory)
 
     def _validate_manifest(self, file, service_manifest_data):
         service_id = service_manifest_data.get('full_service_name')
@@ -70,9 +73,9 @@ class ManifestParser(object):
                     root_directory = self.manifest_directory
 
                 if not isinstance(value, list):
-                    log.error(_("'{0}' section should represent a file"
+                    log.error(_("'{0}' section should represent a file listing"
-                                " listing in manifest {1}".format(
+                                " in manifest {1}".format(root_directory,
-                                root_directory, file)))
+                                                          file)))
                     valid_file_info = False
                     continue
                 for filename in value:
@@ -90,28 +93,29 @@ class ManifestParser(object):
 
     def parse(self):
         manifests = []
-        for file in os.listdir(self.manifest_directory):
+        if os.path.exists(self.manifest_directory):
-            manifest_file = os.path.join(self.manifest_directory, file)
+            for file in os.listdir(self.manifest_directory):
-            if os.path.isfile(manifest_file):
+                manifest_file = os.path.join(self.manifest_directory, file)
-                if not file.endswith(".yaml"):
+                if os.path.isfile(manifest_file):
-                    log.warning(_("Extension of '{0}' file is not yaml. "
+                    if not file.endswith(".yaml"):
-                                  "Only yaml file supported for "
+                        log.warning(_("Extension of {0} file is not yaml. "
-                                  "service manifest files.".format(file)))
+                                      "Only yaml file supported for "
-                    continue
+                                      "service manifest files.".format(file)))
+                        continue
 
-                try:
+                    try:
-                    with open(manifest_file) as stream:
+                        with open(manifest_file) as stream:
-                        manifest_data = yaml.load(stream)
+                            manifest_data = yaml.load(stream)
-                except yaml.YAMLError:
+                    except yaml.YAMLError:
-                    log.exception(_("Failed to load manifest "
+                            log.exception(_("Failed to load manifest file "
-                                    "file '{0}'".format(manifest_file)))
+                                            " '{0}'".format(manifest_file)))
-                    continue
+                            continue
 
-                manifest_is_valid, use_manifest = self._validate_manifest(
+                    manifest_is_valid, use_manifest = self._validate_manifest(
-                    file, manifest_data)
+                        file, manifest_data)
-                if use_manifest:
+                    if use_manifest:
-                    manifest_data["valid"] = manifest_is_valid
+                        manifest_data["valid"] = manifest_is_valid
-                    manifests.append(Manifest(manifest_data))
+                        manifests.append(Manifest(manifest_data))
 
         return manifests
 

muranorepository/utils/utils.py

@@ -0,0 +1,40 @@
+#    Copyright (c) 2013 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import os
+import shutil
+from oslo.config import cfg
+from flask import request
+from muranorepository.consts import SERVICE_DEFINITIONS_FOLDER_NAME
+from muranorepository.consts import CACHE_FOLDER_NAME
+CONF = cfg.CONF
+
+
+def get_tenant_id():
+    return request.environ['keystone.'
+                           'token_info']['access']['token']['tenant']['id']
+
+
+def get_tenant_folder():
+    return os.path.join(CONF.data_dir,
+                        SERVICE_DEFINITIONS_FOLDER_NAME,
+                        get_tenant_id())
+
+
+def get_cache_folder():
+    return os.path.join(CONF.data_dir, CACHE_FOLDER_NAME, get_tenant_id())
+
+
+def check_tenant_dir_existence(path):
+    if not os.path.exists(path):
+        shutil.copytree(CONF.manifests, path)