stackforge-attic/murano-repository
Code Issues Proposed changes

Add per-tenant isolation

Browse Source 
* Fix some exceptions
* use tmpfile to store uploaded archive
Implements blueprint per-tenant-isolation
Change-Id: I9a302823fb208cc73808e573cf4a9812a0c9cf79
This commit is contained in:
Ekaterina Fedorova 2013-12-24 17:02:36 +04:00
parent e83b48becf
commit 9946642d01
8 changed files with 123 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/murano-repository.conf.sample
View File

@ -5,7 +5,7 @@ host = 0.0.0.0
port = 8084
# Directory for cache, OS temp directory is used by default
#cache_dir =
# data_dir = /tmp/muranorepository-data
# Show more verbose log output (sets INFO log level output)
verbose = True

52
muranorepository/api/utils.py
View File

@ -12,6 +12,7 @@ from werkzeug import secure_filename
from muranorepository.utils.parser import ManifestParser
from muranorepository.utils.parser import serialize
from muranorepository.utils.archiver import Archiver
from muranorepository.utils import utils
from muranorepository.consts import DATA_TYPES, MANIFEST
from muranorepository.consts import CLIENTS_DICT
from muranorepository.consts import ARCHIVE_PKG_NAME
@ -23,16 +24,26 @@ CONF = cfg.CONF
def reset_cache():
try:
shutil.rmtree(CONF.cache_dir, ignore_errors=True)
os.mkdir(CONF.cache_dir)
cache_dir = utils.get_cache_folder()
shutil.rmtree(cache_dir, ignore_errors=True)
os.mkdir(cache_dir)
except:
log.exception(_('Error while cleaning cache'))
return make_response('Unable to reset cache', 500)
def compose_path(data_type, path=None):
tenant_dir = utils.get_tenant_folder()
utils.check_tenant_dir_existence(tenant_dir)
return os.path.join(tenant_dir,
getattr(CONF, data_type),
path or '')
def get_archive(client, hash_sum):
types = CLIENTS_DICT.get(client)
archive_manager = Archiver()
cache_dir = os.path.join(CONF.cache_dir, client)
cache_dir = os.path.join(utils.get_cache_folder(), client)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
existing_hash = None
@ -45,6 +56,7 @@ def get_archive(client, hash_sum):
if archive_manager.hashes_match(cache_dir, existing_hash, hash_sum):
return None
manifests = ManifestParser().parse()
return archive_manager.create(cache_dir, manifests, types)
@ -106,13 +118,6 @@ def save_file(request, data_type, path=None, filename=None):
return jsonify(result='success')
def compose_path(data_type, path=None):
if path:
return os.path.join(CONF.manifests, getattr(CONF, data_type), path)
else:
return os.path.join(CONF.manifests, getattr(CONF, data_type))
def check_data_type(data_type):
if data_type not in DATA_TYPES:
abort(404)
@ -147,11 +152,10 @@ def check_service_name(service_name):
def perform_deletion(files_for_deletion, manifest_for_deletion):
def backup_data():
backup_dir = os.path.join(
os.path.dirname(CONF.manifests),
'Backup_{0}'.format(datetime.datetime.utcnow())
)
log.debug(_('Creating service data backup to {0}'.format(backup_dir)))
shutil.copytree(CONF.manifests, backup_dir)
utils.get_cache_folder(),
'Backup_{0}'.format(datetime.datetime.utcnow()))
log.debug('Creating service data backup to {0}'.format(backup_dir))
shutil.copytree(utils.get_tenant_folder(), backup_dir)
return backup_dir
def release_backup(backup):
@ -163,12 +167,12 @@ def perform_deletion(files_for_deletion, manifest_for_deletion):
def restore_backup(backup):
log.debug('Restore service data after unsuccessful deletion')
shutil.rmtree(CONF.manifests, ignore_errors=True)
os.rename(backup, CONF.manifests)
shutil.rmtree(utils.get_tenant_folder(), ignore_errors=True)
os.rename(backup, utils.get_tenant_folder())
backup_dir = backup_data()
service_name = manifest_for_deletion.full_service_name
path_to_manifest = os.path.join(CONF.manifests,
path_to_manifest = os.path.join(utils.get_tenant_folder(),
'{0}-manifest.yaml'.format(service_name))
try:
if os.path.exists(path_to_manifest):
@ -176,8 +180,8 @@ def perform_deletion(files_for_deletion, manifest_for_deletion):
os.remove(path_to_manifest)
for data_type, files in files_for_deletion.iteritems():
data_type_dir = os.path.join(CONF.manifests, getattr(CONF,
data_type))
data_type_dir = os.path.join(utils.get_tenant_folder(),
getattr(CONF, data_type))
for file in files:
path_to_delete = os.path.join(data_type_dir, file)
if os.path.exists(path_to_delete):
@ -205,11 +209,9 @@ def save_archive(request):
path_to_archive = uploaded_file.name
else:
file_to_upload = request.files.get('file')
if file_to_upload:
filename = secure_filename(file_to_upload.filename)
else:
if not file_to_upload:
return err_resp
path_to_archive = os.path.join(CONF.cache_dir, filename)
path_to_archive = tempfile.NamedTemporaryFile(delete=False).name
file_to_upload.save(path_to_archive)
return path_to_archive
@ -230,7 +232,7 @@ def create_or_update_service(service_id, data):
if not data.get(parameter):
data[parameter] = optional[parameter]
path_to_manifest = os.path.join(CONF.manifests,
path_to_manifest = os.path.join(utils.get_tenant_folder(),
service_id + '-manifest.yaml')
backup_done = False

13
muranorepository/cmd/run.py
View File

@ -53,13 +53,14 @@ def main():
log.setup('muranorepository')
#configuring and initializing cache directory
if cfg.CONF.cache_dir is None:
cfg.CONF.cache_dir = os.path.join(
tempfile.gettempdir(), 'muranorepository-cache'
if cfg.CONF.data_dir is None:
cfg.CONF.data_dir = os.path.join(
tempfile.gettempdir(), 'muranorepository-data'
)
if not os.path.exists(cfg.CONF.cache_dir):
os.mkdir(cfg.CONF.cache_dir)
LOG.info('Cache is located at: {0}'.format(cfg.CONF.cache_dir))
if not os.path.exists(cfg.CONF.data_dir):
os.mkdir(cfg.CONF.data_dir)
LOG.info('Cache is located at: {0}'.format(cfg.CONF.data_dir))
app = server.make_app({
'auth_host': cfg.CONF.keystone.auth_host,

2
muranorepository/config.py
View File

@ -21,7 +21,7 @@ server_opts = [
cfg.IntOpt('port', default=5000)
]
cache_opt = cfg.StrOpt('cache_dir')
cache_opt = cfg.StrOpt('data_dir')
keystone_opts = [
cfg.StrOpt('auth_host', default='localhost'),

3
muranorepository/consts.py
View File

@ -30,3 +30,6 @@ ARCHIVE_PKG_NAME = 'data.tar.gz'
UI_FIELDS_IN_MANIFEST = {'description': 'description',
'type': 'full_service_name',
'name': 'service_display_name'}
SERVICE_DEFINITIONS_FOLDER_NAME = 'service_definitions'
CACHE_FOLDER_NAME = 'cache'

27
muranorepository/utils/archiver.py
View File

@ -23,6 +23,7 @@ from oslo.config import cfg
from .parser import serialize
from muranorepository.consts import DATA_TYPES, ARCHIVE_PKG_NAME
from muranorepository.consts import UI, UI_FIELDS_IN_MANIFEST
from muranorepository.utils import utils
from muranorepository.openstack.common.gettextutils import _ # noqa
CONF = cfg.CONF
@ -206,16 +207,17 @@ class Archiver(object):
if hasattr(manifest, data_type):
file_list = getattr(manifest, data_type)
scr_directory = os.path.join(
CONF.manifests, self.src_directories[data_type])
src_directory = os.path.join(
utils.get_tenant_folder(),
self.src_directories[data_type])
dst_directory = os.path.join(
temp_dir, self.dst_directories[data_type])
if data_type == UI:
self._compose_ui_forms(manifest, file_list,
scr_directory, dst_directory)
src_directory, dst_directory)
else:
self._copy_data(file_list,
scr_directory,
src_directory,
dst_directory)
else:
log.info(
@ -234,23 +236,25 @@ class Archiver(object):
for data_type in DATA_TYPES:
if hasattr(manifest, data_type):
file_list = getattr(manifest, data_type)
scr_directory = os.path.join(
CONF.manifests, self.src_directories[data_type])
src_directory = os.path.join(
utils.get_tenant_folder(), self.src_directories[data_type])
dst_directory = os.path.join(
temp_dir, self.dst_directories[data_type])
self._copy_data(file_list, scr_directory, dst_directory)
self._copy_data(file_list, src_directory, dst_directory)
else:
log.info(
_('{0} manifest has no file definitions for '
'{1}'.format(manifest.service_display_name, data_type)))
#Add manifest file into archive
manifest_filename = manifest.full_service_name + '-manifest.yaml'
self._copy_data([manifest_filename], CONF.manifests, temp_dir)
self._copy_data([manifest_filename],
utils.get_tenant_folder(),
temp_dir)
return self._compose_archive(file_name, temp_dir)
def remove_existing_hash(self, cache_dir, hash):
path = os.path.join(cache_dir, hash)
log.info(_('Deleting archive package from {0}.'.format(path)))
log.info('Deleting archive package from {0}.'.format(path))
shutil.rmtree(path, ignore_errors=True)
def extract(self, path_to_archive):
@ -260,6 +264,7 @@ class Archiver(object):
return value - True if succeeded , False otherwise
"""
try:
root_folder = utils.get_tenant_folder()
path_to_extract = tempfile.mkdtemp()
archive = tarfile.open(path_to_archive)
try:
@ -277,7 +282,7 @@ class Archiver(object):
'file in archive'))
return False
shutil.copy(manifests[0], CONF.manifests)
shutil.copy(manifests[0], root_folder)
#Todo: Check manifest is valid
for item in os.listdir(path_to_extract):
item_path = os.path.join(path_to_extract, item)
@ -298,7 +303,7 @@ class Archiver(object):
self._copy_data(file_list,
item_path,
os.path.join(
CONF.manifests,
root_folder,
self.src_directories[item]),
overwrite=False)
else:

52
muranorepository/utils/parser.py
View File

@ -16,9 +16,11 @@ import os
import yaml
from oslo.config import cfg
import logging as log
from muranorepository.manifest import Manifest
from muranorepository.consts import DATA_TYPES, MANIFEST
from muranorepository.openstack.common.gettextutils import _ # noqa
from muranorepository.utils import utils
CONF = cfg.CONF
@ -48,8 +50,9 @@ def serialize(data):
class ManifestParser(object):
def __init__(self, manifest_directory=None):
if not manifest_directory:
manifest_directory = CONF.manifests
manifest_directory = utils.get_tenant_folder()
self.manifest_directory = manifest_directory
utils.check_tenant_dir_existence(self.manifest_directory)
def _validate_manifest(self, file, service_manifest_data):
service_id = service_manifest_data.get('full_service_name')
@ -70,9 +73,9 @@ class ManifestParser(object):
root_directory = self.manifest_directory
if not isinstance(value, list):
log.error(_("'{0}' section should represent a file"
" listing in manifest {1}".format(
root_directory, file)))
log.error(_("'{0}' section should represent a file listing"
" in manifest {1}".format(root_directory,
file)))
valid_file_info = False
continue
for filename in value:
@ -90,28 +93,29 @@ class ManifestParser(object):
def parse(self):
manifests = []
for file in os.listdir(self.manifest_directory):
manifest_file = os.path.join(self.manifest_directory, file)
if os.path.isfile(manifest_file):
if not file.endswith(".yaml"):
log.warning(_("Extension of '{0}' file is not yaml. "
"Only yaml file supported for "
"service manifest files.".format(file)))
continue
if os.path.exists(self.manifest_directory):
for file in os.listdir(self.manifest_directory):
manifest_file = os.path.join(self.manifest_directory, file)
if os.path.isfile(manifest_file):
if not file.endswith(".yaml"):
log.warning(_("Extension of {0} file is not yaml. "
"Only yaml file supported for "
"service manifest files.".format(file)))
continue
try:
with open(manifest_file) as stream:
manifest_data = yaml.load(stream)
except yaml.YAMLError:
log.exception(_("Failed to load manifest "
"file '{0}'".format(manifest_file)))
continue
try:
with open(manifest_file) as stream:
manifest_data = yaml.load(stream)
except yaml.YAMLError:
log.exception(_("Failed to load manifest file "
" '{0}'".format(manifest_file)))
continue
manifest_is_valid, use_manifest = self._validate_manifest(
file, manifest_data)
if use_manifest:
manifest_data["valid"] = manifest_is_valid
manifests.append(Manifest(manifest_data))
manifest_is_valid, use_manifest = self._validate_manifest(
file, manifest_data)
if use_manifest:
manifest_data["valid"] = manifest_is_valid
manifests.append(Manifest(manifest_data))
return manifests

40
muranorepository/utils/utils.py Normal file
View File

@ -0,0 +1,40 @@
# Copyright (c) 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import shutil
from oslo.config import cfg
from flask import request
from muranorepository.consts import SERVICE_DEFINITIONS_FOLDER_NAME
from muranorepository.consts import CACHE_FOLDER_NAME
CONF = cfg.CONF
def get_tenant_id():
return request.environ['keystone.'
'token_info']['access']['token']['tenant']['id']
def get_tenant_folder():
return os.path.join(CONF.data_dir,
SERVICE_DEFINITIONS_FOLDER_NAME,
get_tenant_id())
def get_cache_folder():
return os.path.join(CONF.data_dir, CACHE_FOLDER_NAME, get_tenant_id())
def check_tenant_dir_existence(path):
if not os.path.exists(path):
shutil.copytree(CONF.manifests, path)