From 1d285abb179d247a73a76eadd1705f3a9d18b240 Mon Sep 17 00:00:00 2001 From: LingxianKong Date: Thu, 30 Apr 2015 17:24:40 +0800 Subject: [PATCH] Refactor the whole project structure to make it OpenStack-Style Change-Id: Ideb01dfa0b011143bf301b35e28791b834c598f6 --- neat/config.py | 130 --- {neat => terracotta}/__init__.py | 0 {neat/globals => terracotta/api}/__init__.py | 0 terracotta/api/access_control.py | 51 + terracotta/api/app.py | 62 ++ .../api/controllers}/__init__.py | 0 terracotta/api/controllers/resource.py | 94 ++ terracotta/api/controllers/root.py | 68 ++ .../api/controllers/v2}/__init__.py | 0 terracotta/api/controllers/v2/action.py | 159 +++ .../api/controllers/v2/action_execution.py | 204 ++++ terracotta/api/controllers/v2/cron_trigger.py | 144 +++ terracotta/api/controllers/v2/environment.py | 148 +++ terracotta/api/controllers/v2/execution.py | 203 ++++ terracotta/api/controllers/v2/root.py | 58 ++ terracotta/api/controllers/v2/task.py | 144 +++ terracotta/api/controllers/v2/validation.py | 41 + terracotta/api/controllers/v2/workbook.py | 132 +++ terracotta/api/controllers/v2/workflow.py | 174 ++++ .../api/hooks}/__init__.py | 0 terracotta/api/hooks/content_type.py | 38 + terracotta/api/wsgi.py | 19 + .../mhod => terracotta/cmd}/__init__.py | 0 terracotta/cmd/launch.py | 208 ++++ {neat => terracotta}/common.py | 0 terracotta/config.py | 260 +++++ {neat => terracotta}/contracts_extra.py | 0 {neat => terracotta}/contracts_primitive.py | 0 {neat => terracotta}/db.py | 0 {neat => terracotta}/db_utils.py | 0 .../globals}/__init__.py | 0 {neat => terracotta}/globals/db_cleaner.py | 0 {neat => terracotta}/globals/manager.py | 0 .../globals/vm_placement}/__init__.py | 0 .../globals/vm_placement/bin_packing.py | 0 terracotta/locals/__init__.py | 0 {neat => terracotta}/locals/collector.py | 0 {neat => terracotta}/locals/manager.py | 0 terracotta/locals/overload/__init__.py | 0 terracotta/locals/overload/mhod/__init__.py | 0 .../locals/overload/mhod/bruteforce.py | 0 .../locals/overload/mhod/core.py | 0 .../locals/overload/mhod/l_2_states.py | 0 .../overload/mhod/multisize_estimation.py | 0 .../locals/overload/mhod/nlp.py | 0 {neat => terracotta}/locals/overload/otf.py | 0 .../locals/overload/statistics.py | 0 .../locals/overload/trivial.py | 0 terracotta/locals/underload/__init__.py | 0 .../locals/underload/trivial.py | 0 terracotta/locals/vm_selection/__init__.py | 0 .../locals/vm_selection/algorithms.py | 0 terracotta/openstack/__init__.py | 0 terracotta/openstack/common/README | 16 + terracotta/openstack/common/__init__.py | 0 terracotta/openstack/common/_i18n.py | 45 + terracotta/openstack/common/cliutils.py | 271 +++++ .../openstack/common/config/__init__.py | 0 .../openstack/common/config/generator.py | 320 ++++++ .../openstack/common/eventlet_backdoor.py | 151 +++ terracotta/openstack/common/fileutils.py | 149 +++ terracotta/openstack/common/imageutils.py | 152 +++ terracotta/openstack/common/local.py | 45 + terracotta/openstack/common/loopingcall.py | 147 +++ terracotta/openstack/common/memorycache.py | 97 ++ .../openstack/common/middleware/__init__.py | 0 .../openstack/common/middleware/request_id.py | 27 + terracotta/openstack/common/periodic_task.py | 232 +++++ terracotta/openstack/common/policy.py | 963 ++++++++++++++++++ .../openstack/common/report/__init__.py | 25 + .../common/report/generators/__init__.py | 21 + .../common/report/generators/conf.py | 44 + .../common/report/generators/process.py | 38 + .../common/report/generators/threading.py | 86 ++ .../common/report/generators/version.py | 46 + .../common/report/guru_meditation_report.py | 226 ++++ .../common/report/models/__init__.py | 20 + .../openstack/common/report/models/base.py | 162 +++ .../openstack/common/report/models/conf.py | 66 ++ .../openstack/common/report/models/process.py | 62 ++ .../common/report/models/threading.py | 100 ++ .../openstack/common/report/models/version.py | 44 + .../report/models/with_default_views.py | 81 ++ terracotta/openstack/common/report/report.py | 187 ++++ terracotta/openstack/common/report/utils.py | 46 + .../openstack/common/report/views/__init__.py | 22 + .../common/report/views/jinja_view.py | 137 +++ .../common/report/views/json/__init__.py | 19 + .../common/report/views/json/generic.py | 66 ++ .../common/report/views/text/__init__.py | 19 + .../common/report/views/text/generic.py | 202 ++++ .../common/report/views/text/header.py | 51 + .../common/report/views/text/process.py | 38 + .../common/report/views/text/threading.py | 80 ++ .../common/report/views/xml/__init__.py | 19 + .../common/report/views/xml/generic.py | 87 ++ terracotta/openstack/common/service.py | 509 +++++++++ terracotta/openstack/common/sslutils.py | 81 ++ terracotta/openstack/common/systemd.py | 105 ++ terracotta/openstack/common/threadgroup.py | 149 +++ terracotta/openstack/common/versionutils.py | 262 +++++ terracotta/rpc.py | 395 +++++++ 102 files changed, 8317 insertions(+), 130 deletions(-) delete mode 100644 neat/config.py rename {neat => terracotta}/__init__.py (100%) rename {neat/globals => terracotta/api}/__init__.py (100%) create mode 100644 terracotta/api/access_control.py create mode 100644 terracotta/api/app.py rename {neat/globals/vm_placement => terracotta/api/controllers}/__init__.py (100%) create mode 100644 terracotta/api/controllers/resource.py create mode 100644 terracotta/api/controllers/root.py rename {neat/locals => terracotta/api/controllers/v2}/__init__.py (100%) create mode 100644 terracotta/api/controllers/v2/action.py create mode 100644 terracotta/api/controllers/v2/action_execution.py create mode 100644 terracotta/api/controllers/v2/cron_trigger.py create mode 100644 terracotta/api/controllers/v2/environment.py create mode 100644 terracotta/api/controllers/v2/execution.py create mode 100644 terracotta/api/controllers/v2/root.py create mode 100644 terracotta/api/controllers/v2/task.py create mode 100644 terracotta/api/controllers/v2/validation.py create mode 100644 terracotta/api/controllers/v2/workbook.py create mode 100644 terracotta/api/controllers/v2/workflow.py rename {neat/locals/overload => terracotta/api/hooks}/__init__.py (100%) create mode 100644 terracotta/api/hooks/content_type.py create mode 100644 terracotta/api/wsgi.py rename {neat/locals/overload/mhod => terracotta/cmd}/__init__.py (100%) create mode 100644 terracotta/cmd/launch.py rename {neat => terracotta}/common.py (100%) create mode 100644 terracotta/config.py rename {neat => terracotta}/contracts_extra.py (100%) rename {neat => terracotta}/contracts_primitive.py (100%) rename {neat => terracotta}/db.py (100%) rename {neat => terracotta}/db_utils.py (100%) rename {neat/locals/underload => terracotta/globals}/__init__.py (100%) rename {neat => terracotta}/globals/db_cleaner.py (100%) rename {neat => terracotta}/globals/manager.py (100%) rename {neat/locals/vm_selection => terracotta/globals/vm_placement}/__init__.py (100%) rename {neat => terracotta}/globals/vm_placement/bin_packing.py (100%) create mode 100644 terracotta/locals/__init__.py rename {neat => terracotta}/locals/collector.py (100%) rename {neat => terracotta}/locals/manager.py (100%) create mode 100644 terracotta/locals/overload/__init__.py create mode 100644 terracotta/locals/overload/mhod/__init__.py rename {neat => terracotta}/locals/overload/mhod/bruteforce.py (100%) rename {neat => terracotta}/locals/overload/mhod/core.py (100%) rename {neat => terracotta}/locals/overload/mhod/l_2_states.py (100%) rename {neat => terracotta}/locals/overload/mhod/multisize_estimation.py (100%) rename {neat => terracotta}/locals/overload/mhod/nlp.py (100%) rename {neat => terracotta}/locals/overload/otf.py (100%) rename {neat => terracotta}/locals/overload/statistics.py (100%) rename {neat => terracotta}/locals/overload/trivial.py (100%) create mode 100644 terracotta/locals/underload/__init__.py rename {neat => terracotta}/locals/underload/trivial.py (100%) create mode 100644 terracotta/locals/vm_selection/__init__.py rename {neat => terracotta}/locals/vm_selection/algorithms.py (100%) create mode 100644 terracotta/openstack/__init__.py create mode 100644 terracotta/openstack/common/README create mode 100644 terracotta/openstack/common/__init__.py create mode 100644 terracotta/openstack/common/_i18n.py create mode 100644 terracotta/openstack/common/cliutils.py create mode 100644 terracotta/openstack/common/config/__init__.py create mode 100644 terracotta/openstack/common/config/generator.py create mode 100644 terracotta/openstack/common/eventlet_backdoor.py create mode 100644 terracotta/openstack/common/fileutils.py create mode 100644 terracotta/openstack/common/imageutils.py create mode 100644 terracotta/openstack/common/local.py create mode 100644 terracotta/openstack/common/loopingcall.py create mode 100644 terracotta/openstack/common/memorycache.py create mode 100644 terracotta/openstack/common/middleware/__init__.py create mode 100644 terracotta/openstack/common/middleware/request_id.py create mode 100644 terracotta/openstack/common/periodic_task.py create mode 100644 terracotta/openstack/common/policy.py create mode 100644 terracotta/openstack/common/report/__init__.py create mode 100644 terracotta/openstack/common/report/generators/__init__.py create mode 100644 terracotta/openstack/common/report/generators/conf.py create mode 100644 terracotta/openstack/common/report/generators/process.py create mode 100644 terracotta/openstack/common/report/generators/threading.py create mode 100644 terracotta/openstack/common/report/generators/version.py create mode 100644 terracotta/openstack/common/report/guru_meditation_report.py create mode 100644 terracotta/openstack/common/report/models/__init__.py create mode 100644 terracotta/openstack/common/report/models/base.py create mode 100644 terracotta/openstack/common/report/models/conf.py create mode 100644 terracotta/openstack/common/report/models/process.py create mode 100644 terracotta/openstack/common/report/models/threading.py create mode 100644 terracotta/openstack/common/report/models/version.py create mode 100644 terracotta/openstack/common/report/models/with_default_views.py create mode 100644 terracotta/openstack/common/report/report.py create mode 100644 terracotta/openstack/common/report/utils.py create mode 100644 terracotta/openstack/common/report/views/__init__.py create mode 100644 terracotta/openstack/common/report/views/jinja_view.py create mode 100644 terracotta/openstack/common/report/views/json/__init__.py create mode 100644 terracotta/openstack/common/report/views/json/generic.py create mode 100644 terracotta/openstack/common/report/views/text/__init__.py create mode 100644 terracotta/openstack/common/report/views/text/generic.py create mode 100644 terracotta/openstack/common/report/views/text/header.py create mode 100644 terracotta/openstack/common/report/views/text/process.py create mode 100644 terracotta/openstack/common/report/views/text/threading.py create mode 100644 terracotta/openstack/common/report/views/xml/__init__.py create mode 100644 terracotta/openstack/common/report/views/xml/generic.py create mode 100644 terracotta/openstack/common/service.py create mode 100644 terracotta/openstack/common/sslutils.py create mode 100644 terracotta/openstack/common/systemd.py create mode 100644 terracotta/openstack/common/threadgroup.py create mode 100644 terracotta/openstack/common/versionutils.py create mode 100644 terracotta/rpc.py diff --git a/neat/config.py b/neat/config.py deleted file mode 100644 index 4a92444..0000000 --- a/neat/config.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2012 Anton Beloglazov -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" A set of functions for reading configuration options from files. - -""" - -from contracts import contract -import os -import ConfigParser - -import logging -log = logging.getLogger(__name__) - - -# This is the default config, which should not be modified -DEFAULT_CONFIG_PATH = os.path.join(os.path.dirname(__file__), - '..', - 'neat.conf') - -# This is the custom config, which may override the defaults -CONFIG_PATH = "/etc/neat/neat.conf" -# The following value is used for testing purposes -#CONFIG_PATH = os.path.join(os.path.dirname(__file__), -# '..', -# 'neat.conf') - -# These fields must present in the configuration file -REQUIRED_FIELDS = [ - 'log_directory', - 'log_level', - 'vm_instance_directory', - 'sql_connection', - 'os_admin_tenant_name', - 'os_admin_user', - 'os_admin_password', - 'os_auth_url', - 'compute_hosts', - 'global_manager_host', - 'global_manager_port', - 'db_cleaner_interval', - 'local_data_directory', - 'local_manager_interval', - 'data_collector_interval', - 'data_collector_data_length', - 'host_cpu_overload_threshold', - 'host_cpu_usable_by_vms', - 'compute_user', - 'compute_password', - 'sleep_command', - 'ether_wake_interface', - 'block_migration', - 'network_migration_bandwidth', - 'algorithm_underload_detection_factory', - 'algorithm_underload_detection_parameters', - 'algorithm_overload_detection_factory', - 'algorithm_overload_detection_parameters', - 'algorithm_vm_selection_factory', - 'algorithm_vm_selection_parameters', - 'algorithm_vm_placement_factory', - 'algorithm_vm_placement_parameters', -] - - -@contract -def read_config(paths): - """ Read the configuration files and return the options. - - :param paths: A list of required configuration file paths. - :type paths: list(str) - - :return: A dictionary of the configuration options. - :rtype: dict(str: str) - """ - configParser = ConfigParser.ConfigParser() - for path in paths: - configParser.read(path) - return dict(configParser.items("DEFAULT")) - - -@contract -def validate_config(config, required_fields): - """ Check that the config contains all the required fields. - - :param config: A config dictionary to check. - :type config: dict(str: str) - - :param required_fields: A list of required fields. - :type required_fields: list(str) - - :return: Whether the config is valid. - :rtype: bool - """ - for field in required_fields: - if not field in config: - return False - return True - - -@contract -def read_and_validate_config(paths, required_fields): - """ Read the configuration files, validate and return the options. - - :param paths: A list of required configuration file paths. - :type paths: list(str) - - :param required_fields: A list of required fields. - :type required_fields: list(str) - - :return: A dictionary of the configuration options. - :rtype: dict(str: str) - """ - config = read_config(paths) - if not validate_config(config, required_fields): - message = 'The config dictionary does not contain ' + \ - 'all the required fields' - log.critical(message) - raise KeyError(message) - return config diff --git a/neat/__init__.py b/terracotta/__init__.py similarity index 100% rename from neat/__init__.py rename to terracotta/__init__.py diff --git a/neat/globals/__init__.py b/terracotta/api/__init__.py similarity index 100% rename from neat/globals/__init__.py rename to terracotta/api/__init__.py diff --git a/terracotta/api/access_control.py b/terracotta/api/access_control.py new file mode 100644 index 0000000..c2cd3a7 --- /dev/null +++ b/terracotta/api/access_control.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Access Control API server.""" + +from keystonemiddleware import auth_token +from oslo.config import cfg + + +_ENFORCER = None + + +def setup(app): + if cfg.CONF.pecan.auth_enable: + return auth_token.AuthProtocol(app, dict(cfg.CONF.keystone_authtoken)) + else: + return app + + +def get_limited_to(headers): + """Return the user and project the request should be limited to. + + :param headers: HTTP headers dictionary + :return: A tuple of (user, project), set to None if there's no limit on + one of these. + + """ + return headers.get('X-User-Id'), headers.get('X-Project-Id') + + +def get_limited_to_project(headers): + """Return the project the request should be limited to. + + :param headers: HTTP headers dictionary + :return: A project, or None if there's no limit on it. + + """ + return get_limited_to(headers)[1] diff --git a/terracotta/api/app.py b/terracotta/api/app.py new file mode 100644 index 0000000..6991b10 --- /dev/null +++ b/terracotta/api/app.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.config import cfg +import pecan + +from mistral.api import access_control +from mistral import context as ctx +from mistral.db.v2 import api as db_api_v2 +from mistral.services import periodic + + +def get_pecan_config(): + # Set up the pecan configuration. + opts = cfg.CONF.pecan + + cfg_dict = { + "app": { + "root": opts.root, + "modules": opts.modules, + "debug": opts.debug, + "auth_enable": opts.auth_enable + } + } + + return pecan.configuration.conf_from_dict(cfg_dict) + + +def setup_app(config=None): + if not config: + config = get_pecan_config() + + app_conf = dict(config.app) + + db_api_v2.setup_db() + + periodic.setup() + + app = pecan.make_app( + app_conf.pop('root'), + hooks=lambda: [ctx.ContextHook()], + logging=getattr(config, 'logging', {}), + **app_conf + ) + + # Set up access control. + app = access_control.setup(app) + + return app diff --git a/neat/globals/vm_placement/__init__.py b/terracotta/api/controllers/__init__.py similarity index 100% rename from neat/globals/vm_placement/__init__.py rename to terracotta/api/controllers/__init__.py diff --git a/terracotta/api/controllers/resource.py b/terracotta/api/controllers/resource.py new file mode 100644 index 0000000..eaeeb3c --- /dev/null +++ b/terracotta/api/controllers/resource.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from wsme import types as wtypes + + +class Resource(wtypes.Base): + """REST API Resource.""" + + _wsme_attributes = [] + + def to_dict(self): + d = {} + + for attr in self._wsme_attributes: + attr_val = getattr(self, attr.name) + if not isinstance(attr_val, wtypes.UnsetType): + d[attr.name] = attr_val + + return d + + @classmethod + def from_dict(cls, d): + obj = cls() + + for key, val in d.items(): + if hasattr(obj, key): + setattr(obj, key, val) + + return obj + + def __str__(self): + """WSME based implementation of __str__.""" + + res = "%s [" % type(self).__name__ + + first = True + for attr in self._wsme_attributes: + if not first: + res += ', ' + else: + first = False + + res += "%s='%s'" % (attr.name, getattr(self, attr.name)) + + return res + "]" + + def to_string(self): + return json.dumps(self.to_dict()) + + +class ResourceList(Resource): + """Resource containing the list of other resources.""" + + def to_dict(self): + d = {} + + for attr in self._wsme_attributes: + attr_val = getattr(self, attr.name) + + if isinstance(attr_val, list): + if isinstance(attr_val[0], Resource): + d[attr.name] = [v.to_dict() for v in attr_val] + elif not isinstance(attr_val, wtypes.UnsetType): + d[attr.name] = attr_val + + return d + + +class Link(Resource): + """Web link.""" + + href = wtypes.text + target = wtypes.text + + @classmethod + def sample(cls): + return cls(href='http://example.com/here', + target='here') diff --git a/terracotta/api/controllers/root.py b/terracotta/api/controllers/root.py new file mode 100644 index 0000000..57cfee9 --- /dev/null +++ b/terracotta/api/controllers/root.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import resource +from mistral.api.controllers.v2 import root as v2_root +from mistral.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + +API_STATUS = wtypes.Enum(str, 'SUPPORTED', 'CURRENT', 'DEPRECATED') + + +class APIVersion(resource.Resource): + """API Version.""" + + id = wtypes.text + "The version identifier." + + status = API_STATUS + "The status of the API (SUPPORTED, CURRENT or DEPRECATED)." + + link = resource.Link + "The link to the versioned API." + + @classmethod + def sample(cls): + return cls( + id='v1.0', + status='CURRENT', + link=resource.Link( + target_name='v1', + href='http://example.com:9777/v1' + ) + ) + + +class RootController(object): + v2 = v2_root.Controller() + + @wsme_pecan.wsexpose([APIVersion]) + def index(self): + LOG.debug("Fetching API versions.") + + host_url_v2 = '%s/%s' % (pecan.request.host_url, 'v2') + api_v2 = APIVersion( + id='v2.0', + status='CURRENT', + link=resource.Link(href=host_url_v2, target='v2') + ) + + return [api_v2] diff --git a/neat/locals/__init__.py b/terracotta/api/controllers/v2/__init__.py similarity index 100% rename from neat/locals/__init__.py rename to terracotta/api/controllers/v2/__init__.py diff --git a/terracotta/api/controllers/v2/action.py b/terracotta/api/controllers/v2/action.py new file mode 100644 index 0000000..723b3cc --- /dev/null +++ b/terracotta/api/controllers/v2/action.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +from pecan import hooks +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import resource +from mistral.api.hooks import content_type as ct_hook +from mistral.db.v2 import api as db_api +from mistral import exceptions as exc +from mistral.openstack.common import log as logging +from mistral.services import actions +from mistral.utils import rest_utils + +LOG = logging.getLogger(__name__) +SCOPE_TYPES = wtypes.Enum(str, 'private', 'public') + + +class Action(resource.Resource): + """Action resource. + + NOTE: *name* is immutable. Note that name and description get inferred + from action definition when Mistral service receives a POST request. + So they can't be changed in another way. + + """ + + id = wtypes.text + name = wtypes.text + is_system = bool + input = wtypes.text + + description = wtypes.text + tags = [wtypes.text] + definition = wtypes.text + scope = SCOPE_TYPES + + created_at = wtypes.text + updated_at = wtypes.text + + @classmethod + def sample(cls): + return cls(id='123e4567-e89b-12d3-a456-426655440000', + name='flow', + definition='HERE GOES ACTION DEFINITION IN MISTRAL DSL v2', + tags=['large', 'expensive'], + scope='private', + created_at='1970-01-01T00:00:00.000000', + updated_at='1970-01-01T00:00:00.000000') + + +class Actions(resource.ResourceList): + """A collection of Actions.""" + + actions = [Action] + + @classmethod + def sample(cls): + return cls(actions=[Action.sample()]) + + +class ActionsController(rest.RestController, hooks.HookController): + # TODO(nmakhotkin): Have a discussion with pecan/WSME folks in order + # to have requests and response of different content types. Then + # delete ContentTypeHook. + __hooks__ = [ct_hook.ContentTypeHook("application/json", ['POST', 'PUT'])] + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(Action, wtypes.text) + def get(self, name): + """Return the named action.""" + LOG.info("Fetch action [name=%s]" % name) + + db_model = db_api.get_action_definition(name) + + return Action.from_dict(db_model.to_dict()) + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="text/plain") + def put(self): + """Update one or more actions. + + NOTE: This text is allowed to have definitions + of multiple actions. In this case they all will be updated. + """ + definition = pecan.request.text + LOG.info("Update action(s) [definition=%s]" % definition) + + db_acts = actions.update_actions(definition) + models_dicts = [db_act.to_dict() for db_act in db_acts] + + action_list = [Action.from_dict(act) for act in models_dicts] + + return Actions(actions=action_list).to_string() + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="text/plain") + def post(self): + """Create a new action. + + NOTE: This text is allowed to have definitions + of multiple actions. In this case they all will be created. + """ + definition = pecan.request.text + pecan.response.status = 201 + + LOG.info("Create action(s) [definition=%s]" % definition) + + db_acts = actions.create_actions(definition) + models_dicts = [db_act.to_dict() for db_act in db_acts] + + action_list = [Action.from_dict(act) for act in models_dicts] + + return Actions(actions=action_list).to_string() + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(None, wtypes.text, status_code=204) + def delete(self, name): + """Delete the named action.""" + LOG.info("Delete action [name=%s]" % name) + + with db_api.transaction(): + db_model = db_api.get_action_definition(name) + + if db_model.is_system: + msg = "Attempt to delete a system action: %s" % name + raise exc.DataAccessException(msg) + + db_api.delete_action_definition(name) + + @wsme_pecan.wsexpose(Actions) + def get_all(self): + """Return all actions. + + Where project_id is the same as the requester or + project_id is different but the scope is public. + """ + LOG.info("Fetch actions.") + + action_list = [Action.from_dict(db_model.to_dict()) + for db_model in db_api.get_action_definitions()] + + return Actions(actions=action_list) diff --git a/terracotta/api/controllers/v2/action_execution.py b/terracotta/api/controllers/v2/action_execution.py new file mode 100644 index 0000000..2604fb6 --- /dev/null +++ b/terracotta/api/controllers/v2/action_execution.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2015 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import resource +from mistral.db.v2 import api as db_api +from mistral.engine import rpc +from mistral import exceptions as exc +from mistral.openstack.common import log as logging +from mistral.utils import rest_utils +from mistral.workflow import states +from mistral.workflow import utils as wf_utils + + +LOG = logging.getLogger(__name__) + + +class ActionExecution(resource.Resource): + """ActionExecution resource.""" + + id = wtypes.text + + workflow_name = wtypes.text + task_name = wtypes.text + task_execution_id = wtypes.text + + state = wtypes.text + + state_info = wtypes.text + tags = [wtypes.text] + name = wtypes.text + accepted = bool + input = wtypes.text + output = wtypes.text + created_at = wtypes.text + updated_at = wtypes.text + + @classmethod + def from_dict(cls, d): + e = cls() + + for key, val in d.items(): + if hasattr(e, key): + # Nonetype check for dictionary must be explicit. + if val is not None and ( + key == 'input' or key == 'output'): + val = json.dumps(val) + setattr(e, key, val) + + return e + + @classmethod + def sample(cls): + return cls( + id='123e4567-e89b-12d3-a456-426655440000', + workflow_name='flow', + task_name='task1', + workflow_execution_id='653e4127-e89b-12d3-a456-426655440076', + task_execution_id='343e45623-e89b-12d3-a456-426655440090', + state=states.SUCCESS, + state_info=states.SUCCESS, + tags=['foo', 'fee'], + definition_name='std.echo', + accepted=True, + input='{"first_name": "John", "last_name": "Doe"}', + output='{"some_output": "Hello, John Doe!"}', + created_at='1970-01-01T00:00:00.000000', + updated_at='1970-01-01T00:00:00.000000' + ) + + +class ActionExecutions(resource.Resource): + """A collection of action_executions.""" + + action_executions = [ActionExecution] + + @classmethod + def sample(cls): + return cls(action_executions=[ActionExecution.sample()]) + + +def _load_deferred_output_field(action_ex): + # We need to refer to this lazy-load field explicitly in + # order to make sure that it is correctly loaded. + hasattr(action_ex, 'output') + + +def _get_action_execution(id): + action_ex = db_api.get_action_execution(id) + + return _get_action_execution_resource(action_ex) + + +def _get_action_execution_resource(action_ex): + _load_deferred_output_field(action_ex) + + # TODO(nmakhotkin): Get rid of using dicts for constructing resources. + # TODO(nmakhotkin): Use db_model for this instead. + res = ActionExecution.from_dict(action_ex.to_dict()) + + setattr(res, 'task_name', action_ex.task_execution.name) + + return res + + +def _get_action_executions(task_execution_id=None): + kwargs = {'type': 'action_execution'} + + if task_execution_id: + kwargs['task_execution_id'] = task_execution_id + + action_executions = [] + + for action_ex in db_api.get_action_executions(**kwargs): + action_executions.append( + _get_action_execution_resource(action_ex) + ) + + return ActionExecutions(action_executions=action_executions) + + +class ActionExecutionsController(rest.RestController): + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(ActionExecution, wtypes.text) + def get(self, id): + """Return the specified action_execution.""" + LOG.info("Fetch action_execution [id=%s]" % id) + + return _get_action_execution(id) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(ActionExecution, wtypes.text, body=ActionExecution) + def put(self, id, action_execution): + """Update the specified action_execution.""" + LOG.info( + "Update action_execution [id=%s, action_execution=%s]" + % (id, action_execution) + ) + + # Client must provide a valid json. It shouldn't necessarily be an + # object but it should be json complaint so strings have to be escaped. + output = None + + if action_execution.output: + try: + output = json.loads(action_execution.output) + except (ValueError, TypeError) as e: + raise exc.InvalidResultException(str(e)) + + if action_execution.state == states.SUCCESS: + result = wf_utils.Result(data=output) + elif action_execution.state == states.ERROR: + result = wf_utils.Result(error=output) + else: + raise exc.InvalidResultException( + "Error. Expected on of %s, actual: %s" % + ([states.SUCCESS, states.ERROR], action_execution.state) + ) + + values = rpc.get_engine_client().on_action_complete(id, result) + + return ActionExecution.from_dict(values) + + @wsme_pecan.wsexpose(ActionExecutions) + def get_all(self): + """Return all action_executions within the execution.""" + LOG.info("Fetch action_executions") + + return _get_action_executions() + + +class TasksActionExecutionController(rest.RestController): + @wsme_pecan.wsexpose(ActionExecutions, wtypes.text) + def get_all(self, task_execution_id): + """Return all action executions within the task execution.""" + LOG.info("Fetch action executions") + + return _get_action_executions(task_execution_id=task_execution_id) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(ActionExecution, wtypes.text, wtypes.text) + def get(self, task_execution_id, action_ex_id): + """Return the specified action_execution.""" + LOG.info("Fetch action_execution [id=%s]" % action_ex_id) + + return _get_action_execution(action_ex_id) diff --git a/terracotta/api/controllers/v2/cron_trigger.py b/terracotta/api/controllers/v2/cron_trigger.py new file mode 100644 index 0000000..2602940 --- /dev/null +++ b/terracotta/api/controllers/v2/cron_trigger.py @@ -0,0 +1,144 @@ +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import resource +from mistral.db.v2 import api as db_api +from mistral.openstack.common import log as logging +from mistral.services import triggers +from mistral.utils import rest_utils + +LOG = logging.getLogger(__name__) +SCOPE_TYPES = wtypes.Enum(str, 'private', 'public') + + +class CronTrigger(resource.Resource): + """CronTrigger resource.""" + + id = wtypes.text + name = wtypes.text + workflow_name = wtypes.text + workflow_input = wtypes.text + + scope = SCOPE_TYPES + + pattern = wtypes.text + remaining_executions = wtypes.IntegerType(minimum=1) + first_execution_time = wtypes.text + next_execution_time = wtypes.text + + created_at = wtypes.text + updated_at = wtypes.text + + def to_dict(self): + d = super(CronTrigger, self).to_dict() + + if d.get('workflow_input'): + d['workflow_input'] = json.loads(d['workflow_input']) + + return d + + @classmethod + def from_dict(cls, d): + e = cls() + + for key, val in d.items(): + if hasattr(e, key): + # Nonetype check for dictionary must be explicit. + if key == 'workflow_input' and val is not None: + val = json.dumps(val) + + setattr(e, key, val) + + return e + + @classmethod + def sample(cls): + return cls(id='123e4567-e89b-12d3-a456-426655440000', + name='my_trigger', + workflow_name='my_wf', + workflow_input={}, + scope='private', + pattern='* * * * *', + remaining_executions=42, + created_at='1970-01-01T00:00:00.000000', + updated_at='1970-01-01T00:00:00.000000') + + +class CronTriggers(resource.Resource): + """A collection of cron triggers.""" + + cron_triggers = [CronTrigger] + + @classmethod + def sample(cls): + return cls(cron_triggers=[CronTrigger.sample()]) + + +class CronTriggersController(rest.RestController): + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(CronTrigger, wtypes.text) + def get(self, name): + """Returns the named cron_trigger.""" + + LOG.info('Fetch cron trigger [name=%s]' % name) + + db_model = db_api.get_cron_trigger(name) + + return CronTrigger.from_dict(db_model.to_dict()) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(CronTrigger, body=CronTrigger, status_code=201) + def post(self, cron_trigger): + """Creates a new cron trigger.""" + + LOG.info('Create cron trigger: %s' % cron_trigger) + + values = cron_trigger.to_dict() + + db_model = triggers.create_cron_trigger( + values['name'], + values['workflow_name'], + values.get('workflow_input'), + values.get('pattern'), + values.get('first_execution_time'), + values.get('remaining_executions') + ) + + return CronTrigger.from_dict(db_model.to_dict()) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(None, wtypes.text, status_code=204) + def delete(self, name): + """Delete cron trigger.""" + LOG.info("Delete cron trigger [name=%s]" % name) + + db_api.delete_cron_trigger(name) + + @wsme_pecan.wsexpose(CronTriggers) + def get_all(self): + """Return all cron triggers.""" + + LOG.info("Fetch cron triggers.") + + _list = [ + CronTrigger.from_dict(db_model.to_dict()) + for db_model in db_api.get_cron_triggers() + ] + + return CronTriggers(cron_triggers=_list) diff --git a/terracotta/api/controllers/v2/environment.py b/terracotta/api/controllers/v2/environment.py new file mode 100644 index 0000000..2f710d9 --- /dev/null +++ b/terracotta/api/controllers/v2/environment.py @@ -0,0 +1,148 @@ +# Copyright 2015 - StackStorm, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import uuid + +from pecan import rest +import six +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import resource +from mistral.db.v2 import api as db_api +from mistral.openstack.common import log as logging +from mistral.utils import rest_utils + + +LOG = logging.getLogger(__name__) + +SAMPLE = { + 'server': 'localhost', + 'database': 'temp', + 'timeout': 600, + 'verbose': True +} + + +class Environment(resource.Resource): + """Environment resource.""" + + id = wtypes.text + name = wtypes.text + description = wtypes.text + variables = wtypes.text + scope = wtypes.Enum(str, 'private', 'public') + created_at = wtypes.text + updated_at = wtypes.text + + def __init__(self, *args, **kwargs): + super(Environment, self).__init__() + + for key, val in six.iteritems(kwargs): + if key == 'variables' and val is not None: + val = json.dumps(val) + + setattr(self, key, val) + + def to_dict(self): + d = super(Environment, self).to_dict() + + if d.get('variables'): + d['variables'] = json.loads(d['variables']) + + return d + + @classmethod + def from_dict(cls, d): + return cls(**d) + + @classmethod + def sample(cls): + return cls(id=str(uuid.uuid4()), + name='sample', + description='example environment entry', + variables=json.dumps(SAMPLE), + scope='private', + created_at='1970-01-01T00:00:00.000000', + updated_at='1970-01-01T00:00:00.000000') + + +class Environments(resource.Resource): + """A collection of Environment resources.""" + + environments = [Environment] + + @classmethod + def sample(cls): + return cls(environments=[Environment.sample()]) + + +class EnvironmentController(rest.RestController): + + @wsme_pecan.wsexpose(Environments) + def get_all(self): + """Return all environments. + Where project_id is the same as the requestor or + project_id is different but the scope is public. + """ + LOG.info("Fetch environments.") + + environments = [Environment(**db_model.to_dict()) + for db_model in db_api.get_environments()] + + return Environments(environments=environments) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(Environment, wtypes.text) + def get(self, name): + """Return the named environment.""" + LOG.info("Fetch environment [name=%s]" % name) + + db_model = db_api.get_environment(name) + + return Environment(**db_model.to_dict()) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(Environment, body=Environment, status_code=201) + def post(self, environment): + """Create a new environment.""" + LOG.info("Create environment [env=%s]" % environment) + + db_model = db_api.create_environment(environment.to_dict()) + + return Environment(**db_model.to_dict()) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(Environment, body=Environment) + def put(self, environment): + """Update an environment.""" + if not environment.name: + raise ValueError('Name of the environment is not provided.') + + LOG.info("Update environment [name=%s, env=%s]" % + (environment.name, environment)) + + db_model = db_api.update_environment(environment.name, + environment.to_dict()) + + return Environment(**db_model.to_dict()) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(None, wtypes.text, status_code=204) + def delete(self, name): + """Delete the named environment.""" + LOG.info("Delete environment [name=%s]" % name) + + db_api.delete_environment(name) diff --git a/terracotta/api/controllers/v2/execution.py b/terracotta/api/controllers/v2/execution.py new file mode 100644 index 0000000..a4288c1 --- /dev/null +++ b/terracotta/api/controllers/v2/execution.py @@ -0,0 +1,203 @@ +# Copyright 2013 - Mirantis, Inc. +# Copyright 2015 - StackStorm, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import resource +from mistral.api.controllers.v2 import task +from mistral.db.v2 import api as db_api +from mistral.engine import rpc +from mistral import exceptions as exc +from mistral.openstack.common import log as logging +from mistral.utils import rest_utils +from mistral.workflow import states + + +LOG = logging.getLogger(__name__) + +# TODO(rakhmerov): Make sure to make all needed renaming on public API. + + +class Execution(resource.Resource): + """Execution resource.""" + + id = wtypes.text + "id is immutable and auto assigned." + + workflow_name = wtypes.text + "reference to workflow definition" + + params = wtypes.text + "params define workflow type specific parameters. For example, reverse \ + workflow takes one parameter 'task_name' that defines a target task." + + state = wtypes.text + "state can be one of: RUNNING, SUCCESS, ERROR, PAUSED" + + state_info = wtypes.text + "an optional state information string" + + input = wtypes.text + "input is a JSON structure containing workflow input values." + output = wtypes.text + "output is a workflow output." + + created_at = wtypes.text + updated_at = wtypes.text + + # Context is a JSON object but since WSME doesn't support arbitrary + # dictionaries we have to use text type convert to json and back manually. + def to_dict(self): + d = super(Execution, self).to_dict() + + if d.get('input'): + d['input'] = json.loads(d['input']) + + if d.get('output'): + d['output'] = json.loads(d['output']) + + if d.get('params'): + d['params'] = json.loads(d['params']) + + return d + + @classmethod + def from_dict(cls, d): + e = cls() + + for key, val in d.items(): + if hasattr(e, key): + # Nonetype check for dictionary must be explicit + if key in ['input', 'output', 'params'] and val is not None: + val = json.dumps(val) + setattr(e, key, val) + + return e + + @classmethod + def sample(cls): + return cls(id='123e4567-e89b-12d3-a456-426655440000', + workflow_name='flow', + state='SUCCESS', + input='{}', + output='{}', + params='{"env": {"k1": "abc", "k2": 123}}', + created_at='1970-01-01T00:00:00.000000', + updated_at='1970-01-01T00:00:00.000000') + + +class Executions(resource.Resource): + """A collection of Execution resources.""" + + executions = [Execution] + + @classmethod + def sample(cls): + return cls(executions=[Execution.sample()]) + + +class ExecutionsController(rest.RestController): + tasks = task.ExecutionTasksController() + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(Execution, wtypes.text) + def get(self, id): + """Return the specified Execution.""" + LOG.info("Fetch execution [id=%s]" % id) + + return Execution.from_dict(db_api.get_workflow_execution(id).to_dict()) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(Execution, wtypes.text, body=Execution) + def put(self, id, execution): + """Update the specified Execution. + + :param id: execution ID. + :param execution: Execution objects + """ + LOG.info("Update execution [id=%s, execution=%s]" % + (id, execution)) + db_api.ensure_workflow_execution_exists(id) + + # Currently we can change only state. + if not execution.state: + raise exc.DataAccessException( + "Only state of execution can change. " + "Missing 'state' property." + ) + + new_state = execution.state + msg = execution.state_info + + if new_state == states.PAUSED: + wf_ex = rpc.get_engine_client().pause_workflow(id) + elif new_state == states.RUNNING: + wf_ex = rpc.get_engine_client().resume_workflow(id) + elif new_state in [states.SUCCESS, states.ERROR]: + wf_ex = rpc.get_engine_client().stop_workflow(id, new_state, msg) + else: + # To prevent changing state in other cases throw a message. + raise exc.DataAccessException( + "Can not change state to %s. Allowed states are: '%s" % + (new_state, ", ".join([states.RUNNING, states.PAUSED, + states.SUCCESS, states.ERROR])) + ) + + return Execution.from_dict( + wf_ex if isinstance(wf_ex, dict) else wf_ex.to_dict() + ) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(Execution, body=Execution, status_code=201) + def post(self, execution): + """Create a new Execution. + + :param execution: Execution object with input content. + """ + LOG.info("Create execution [execution=%s]" % execution) + + engine = rpc.get_engine_client() + exec_dict = execution.to_dict() + + result = engine.start_workflow( + exec_dict['workflow_name'], + exec_dict.get('input'), + **exec_dict.get('params') or {} + ) + + return Execution.from_dict(result) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(None, wtypes.text, status_code=204) + def delete(self, id): + """Delete the specified Execution.""" + LOG.info("Delete execution [id=%s]" % id) + + return db_api.delete_workflow_execution(id) + + @wsme_pecan.wsexpose(Executions) + def get_all(self): + """Return all Executions.""" + LOG.info("Fetch executions") + + wf_executions = [ + Execution.from_dict(db_model.to_dict()) + for db_model in db_api.get_workflow_executions() + ] + + return Executions(executions=wf_executions) diff --git a/terracotta/api/controllers/v2/root.py b/terracotta/api/controllers/v2/root.py new file mode 100644 index 0000000..1c230ad --- /dev/null +++ b/terracotta/api/controllers/v2/root.py @@ -0,0 +1,58 @@ +# Copyright 2013 - Mirantis, Inc. +# Copyright 2015 - StackStorm, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import resource +from mistral.api.controllers.v2 import action +from mistral.api.controllers.v2 import action_execution +from mistral.api.controllers.v2 import cron_trigger +from mistral.api.controllers.v2 import environment +from mistral.api.controllers.v2 import execution +from mistral.api.controllers.v2 import task +from mistral.api.controllers.v2 import workbook +from mistral.api.controllers.v2 import workflow + + +class RootResource(resource.Resource): + """Root resource for API version 2. + + It references all other resources belonging to the API. + """ + + uri = wtypes.text + + # TODO(everyone): what else do we need here? + # TODO(everyone): we need to collect all the links from API v2.0 + # and provide them. + + +class Controller(object): + """API root controller for version 2.""" + + workbooks = workbook.WorkbooksController() + actions = action.ActionsController() + workflows = workflow.WorkflowsController() + executions = execution.ExecutionsController() + tasks = task.TasksController() + cron_triggers = cron_trigger.CronTriggersController() + environments = environment.EnvironmentController() + action_executions = action_execution.ActionExecutionsController() + + @wsme_pecan.wsexpose(RootResource) + def index(self): + return RootResource(uri='%s/%s' % (pecan.request.host_url, 'v2')) diff --git a/terracotta/api/controllers/v2/task.py b/terracotta/api/controllers/v2/task.py new file mode 100644 index 0000000..2e35011 --- /dev/null +++ b/terracotta/api/controllers/v2/task.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2013 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import resource +from mistral.api.controllers.v2 import action_execution +from mistral.db.v2 import api as db_api +from mistral.openstack.common import log as logging +from mistral.utils import rest_utils +from mistral.workflow import data_flow +from mistral.workflow import states + + +LOG = logging.getLogger(__name__) + + +class Task(resource.Resource): + """Task resource.""" + + id = wtypes.text + name = wtypes.text + + workflow_name = wtypes.text + workflow_execution_id = wtypes.text + + state = wtypes.text + "state can take one of the following values: \ + IDLE, RUNNING, SUCCESS, ERROR, DELAYED" + + result = wtypes.text + published = wtypes.text + + created_at = wtypes.text + updated_at = wtypes.text + + @classmethod + def from_dict(cls, d): + e = cls() + + for key, val in d.items(): + if hasattr(e, key): + # Nonetype check for dictionary must be explicit. + if val is not None and key == 'published': + val = json.dumps(val) + setattr(e, key, val) + + return e + + @classmethod + def sample(cls): + return cls( + id='123e4567-e89b-12d3-a456-426655440000', + workflow_name='flow', + workflow_execution_id='123e4567-e89b-12d3-a456-426655440000', + name='task', + description='tell when you are done', + state=states.SUCCESS, + tags=['foo', 'fee'], + input='{"first_name": "John", "last_name": "Doe"}', + output='{"task": {"build_greeting": ' + '{"greeting": "Hello, John Doe!"}}}', + created_at='1970-01-01T00:00:00.000000', + updated_at='1970-01-01T00:00:00.000000' + ) + + +class Tasks(resource.Resource): + """A collection of tasks.""" + + tasks = [Task] + + @classmethod + def sample(cls): + return cls(tasks=[Task.sample()]) + + +def _get_task_resources_with_results(wf_ex_id=None): + filters = {} + + if wf_ex_id: + filters['workflow_execution_id'] = wf_ex_id + + tasks = [] + task_execs = db_api.get_task_executions(**filters) + for task_ex in task_execs: + task = Task.from_dict(task_ex.to_dict()) + task.result = json.dumps( + data_flow.get_task_execution_result(task_ex) + ) + + tasks += [task] + + return Tasks(tasks=tasks) + + +class TasksController(rest.RestController): + action_executions = action_execution.TasksActionExecutionController() + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(Task, wtypes.text) + def get(self, id): + """Return the specified task.""" + LOG.info("Fetch task [id=%s]" % id) + + task_ex = db_api.get_task_execution(id) + task = Task.from_dict(task_ex.to_dict()) + + task.result = json.dumps(data_flow.get_task_execution_result(task_ex)) + + return task + + @wsme_pecan.wsexpose(Tasks) + def get_all(self): + """Return all tasks within the execution.""" + LOG.info("Fetch tasks") + + return _get_task_resources_with_results() + + +class ExecutionTasksController(rest.RestController): + @wsme_pecan.wsexpose(Tasks, wtypes.text) + def get_all(self, workflow_execution_id): + """Return all tasks within the workflow execution.""" + LOG.info("Fetch tasks") + + return _get_task_resources_with_results(workflow_execution_id) diff --git a/terracotta/api/controllers/v2/validation.py b/terracotta/api/controllers/v2/validation.py new file mode 100644 index 0000000..7622e22 --- /dev/null +++ b/terracotta/api/controllers/v2/validation.py @@ -0,0 +1,41 @@ +# Copyright 2015 - StackStorm, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +from pecan import rest + +from mistral import exceptions as exc +from mistral.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +class SpecValidationController(rest.RestController): + + def __init__(self, parser): + super(SpecValidationController, self).__init__() + self._parse_func = parser + + @pecan.expose('json') + def post(self): + """Validate a spec.""" + definition = pecan.request.text + + try: + self._parse_func(definition) + except exc.DSLParsingException as e: + return {'valid': False, 'error': e.message} + + return {'valid': True} diff --git a/terracotta/api/controllers/v2/workbook.py b/terracotta/api/controllers/v2/workbook.py new file mode 100644 index 0000000..7ca8142 --- /dev/null +++ b/terracotta/api/controllers/v2/workbook.py @@ -0,0 +1,132 @@ +# Copyright 2013 - Mirantis, Inc. +# Copyright 2015 - StackStorm, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +from pecan import hooks +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import resource +from mistral.api.controllers.v2 import validation +from mistral.api.hooks import content_type as ct_hook +from mistral.db.v2 import api as db_api +from mistral.openstack.common import log as logging +from mistral.services import workbooks +from mistral.utils import rest_utils +from mistral.workbook import parser as spec_parser + + +LOG = logging.getLogger(__name__) +SCOPE_TYPES = wtypes.Enum(str, 'private', 'public') + + +class Workbook(resource.Resource): + """Workbook resource.""" + + id = wtypes.text + name = wtypes.text + + definition = wtypes.text + "workbook definition in Mistral v2 DSL" + tags = [wtypes.text] + scope = SCOPE_TYPES + "'private' or 'public'" + + created_at = wtypes.text + updated_at = wtypes.text + + @classmethod + def sample(cls): + return cls(id='123e4567-e89b-12d3-a456-426655440000', + name='book', + definition='HERE GOES' + 'WORKBOOK DEFINITION IN MISTRAL DSL v2', + tags=['large', 'expensive'], + scope='private', + created_at='1970-01-01T00:00:00.000000', + updated_at='1970-01-01T00:00:00.000000') + + +class Workbooks(resource.Resource): + """A collection of Workbooks.""" + + workbooks = [Workbook] + + @classmethod + def sample(cls): + return cls(workbooks=[Workbook.sample()]) + + +class WorkbooksController(rest.RestController, hooks.HookController): + __hooks__ = [ct_hook.ContentTypeHook("application/json", ['POST', 'PUT'])] + + validate = validation.SpecValidationController( + spec_parser.get_workbook_spec_from_yaml) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(Workbook, wtypes.text) + def get(self, name): + """Return the named workbook.""" + LOG.info("Fetch workbook [name=%s]" % name) + + db_model = db_api.get_workbook(name) + + return Workbook.from_dict(db_model.to_dict()) + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="text/plain") + def put(self): + """Update a workbook.""" + definition = pecan.request.text + LOG.info("Update workbook [definition=%s]" % definition) + + wb_db = workbooks.update_workbook_v2(definition) + + return Workbook.from_dict(wb_db.to_dict()).to_string() + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="text/plain") + def post(self): + """Create a new workbook.""" + definition = pecan.request.text + LOG.info("Create workbook [definition=%s]" % definition) + + wb_db = workbooks.create_workbook_v2(definition) + pecan.response.status = 201 + + return Workbook.from_dict(wb_db.to_dict()).to_string() + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(None, wtypes.text, status_code=204) + def delete(self, name): + """Delete the named workbook.""" + LOG.info("Delete workbook [name=%s]" % name) + + db_api.delete_workbook(name) + + @wsme_pecan.wsexpose(Workbooks) + def get_all(self): + """Return all workbooks. + + Where project_id is the same as the requestor or + project_id is different but the scope is public. + """ + LOG.info("Fetch workbooks.") + + workbooks_list = [Workbook.from_dict(db_model.to_dict()) + for db_model in db_api.get_workbooks()] + + return Workbooks(workbooks=workbooks_list) diff --git a/terracotta/api/controllers/v2/workflow.py b/terracotta/api/controllers/v2/workflow.py new file mode 100644 index 0000000..54b2ec5 --- /dev/null +++ b/terracotta/api/controllers/v2/workflow.py @@ -0,0 +1,174 @@ +# Copyright 2013 - Mirantis, Inc. +# Copyright 2015 - StackStorm, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +from pecan import hooks +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from mistral.api.controllers import resource +from mistral.api.controllers.v2 import validation +from mistral.api.hooks import content_type as ct_hook +from mistral.db.v2 import api as db_api +from mistral.openstack.common import log as logging +from mistral.services import workflows +from mistral.utils import rest_utils +from mistral.workbook import parser as spec_parser + + +LOG = logging.getLogger(__name__) +SCOPE_TYPES = wtypes.Enum(str, 'private', 'public') + + +class Workflow(resource.Resource): + """Workflow resource.""" + + id = wtypes.text + name = wtypes.text + input = wtypes.text + + definition = wtypes.text + "Workflow definition in Mistral v2 DSL" + tags = [wtypes.text] + scope = SCOPE_TYPES + "'private' or 'public'" + + created_at = wtypes.text + updated_at = wtypes.text + + @classmethod + def sample(cls): + return cls(id='123e4567-e89b-12d3-a456-426655440000', + name='flow', + input='param1, param2', + definition='HERE GOES' + 'WORKFLOW DEFINITION IN MISTRAL DSL v2', + tags=['large', 'expensive'], + scope='private', + created_at='1970-01-01T00:00:00.000000', + updated_at='1970-01-01T00:00:00.000000') + + @classmethod + def from_dict(cls, d): + e = cls() + input_list = [] + + for key, val in d.items(): + if hasattr(e, key): + setattr(e, key, val) + + input = d['spec'].get('input', []) + for param in input: + if isinstance(param, dict): + for k, v in param.items(): + input_list.append("%s=%s" % (k, v)) + else: + input_list.append(param) + + setattr(e, 'input', ", ".join(input_list) if input_list else None) + + return e + + +class Workflows(resource.ResourceList): + """A collection of workflows.""" + + workflows = [Workflow] + + @classmethod + def sample(cls): + return cls(workflows=[Workflow.sample()]) + + +class WorkflowsController(rest.RestController, hooks.HookController): + # TODO(nmakhotkin): Have a discussion with pecan/WSME folks in order + # to have requests and response of different content types. Then + # delete ContentTypeHook. + __hooks__ = [ct_hook.ContentTypeHook("application/json", ['POST', 'PUT'])] + + validate = validation.SpecValidationController( + spec_parser.get_workflow_list_spec_from_yaml) + + @rest_utils.wrap_wsme_controller_exception + @wsme_pecan.wsexpose(Workflow, wtypes.text) + def get(self, name): + """Return the named workflow.""" + LOG.info("Fetch workflow [name=%s]" % name) + + db_model = db_api.get_workflow_definition(name) + + return Workflow.from_dict(db_model.to_dict()) + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="text/plain") + def put(self): + """Update one or more workflows. + + NOTE: The text is allowed to have definitions + of multiple workflows. In this case they all will be updated. + """ + definition = pecan.request.text + + LOG.info("Update workflow(s) [definition=%s]" % definition) + + db_wfs = workflows.update_workflows(definition) + models_dicts = [db_wf.to_dict() for db_wf in db_wfs] + + workflow_list = [Workflow.from_dict(wf) for wf in models_dicts] + + return Workflows(workflows=workflow_list).to_string() + + @rest_utils.wrap_pecan_controller_exception + @pecan.expose(content_type="text/plain") + def post(self): + """Create a new workflow. + + NOTE: The text is allowed to have definitions + of multiple workflows. In this case they all will be created. + """ + definition = pecan.request.text + pecan.response.status = 201 + + LOG.info("Create workflow(s) [definition=%s]" % definition) + + db_wfs = workflows.create_workflows(definition) + models_dicts = [db_wf.to_dict() for db_wf in db_wfs] + + workflow_list = [Workflow.from_dict(wf) for wf in models_dicts] + + return Workflows(workflows=workflow_list).to_string() + + @rest_utils.wrap_pecan_controller_exception + @wsme_pecan.wsexpose(None, wtypes.text, status_code=204) + def delete(self, name): + """Delete the named workflow.""" + LOG.info("Delete workflow [name=%s]" % name) + + db_api.delete_workflow_definition(name) + + @wsme_pecan.wsexpose(Workflows) + def get_all(self): + """Return all workflows. + + Where project_id is the same as the requester or + project_id is different but the scope is public. + """ + LOG.info("Fetch workflows.") + + workflows_list = [Workflow.from_dict(db_model.to_dict()) + for db_model in db_api.get_workflow_definitions()] + + return Workflows(workflows=workflows_list) diff --git a/neat/locals/overload/__init__.py b/terracotta/api/hooks/__init__.py similarity index 100% rename from neat/locals/overload/__init__.py rename to terracotta/api/hooks/__init__.py diff --git a/terracotta/api/hooks/content_type.py b/terracotta/api/hooks/content_type.py new file mode 100644 index 0000000..029b757 --- /dev/null +++ b/terracotta/api/hooks/content_type.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2014 - Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pecan import hooks + + +class ContentTypeHook(hooks.PecanHook): + def __init__(self, content_type, methods=['GET']): + """Content type hook is needed for changing content type of + responses but only for some HTTP methods. This is kind of + 'hack' but it seems impossible using pecan/WSME to set different + content types on request and response. + + :param content_type: Content-Type that response should has. + :type content_type: str + :param methods: HTTP methods that should have response + with given content_type. + :type methods: list + """ + self.content_type = content_type + self.methods = methods + + def after(self, state): + if state.request.method in self.methods: + state.response.content_type = self.content_type diff --git a/terracotta/api/wsgi.py b/terracotta/api/wsgi.py new file mode 100644 index 0000000..f387f7f --- /dev/null +++ b/terracotta/api/wsgi.py @@ -0,0 +1,19 @@ +# Copyright 2015 - StackStorm, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mistral.api import app +from mistral import config + +config.parse_args() +application = app.setup_app() diff --git a/neat/locals/overload/mhod/__init__.py b/terracotta/cmd/__init__.py similarity index 100% rename from neat/locals/overload/mhod/__init__.py rename to terracotta/cmd/__init__.py diff --git a/terracotta/cmd/launch.py b/terracotta/cmd/launch.py new file mode 100644 index 0000000..10d8d73 --- /dev/null +++ b/terracotta/cmd/launch.py @@ -0,0 +1,208 @@ +# Copyright 2015 - Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +import eventlet + +eventlet.monkey_patch( + os=True, + select=True, + socket=True, + thread=False if '--use-debugger' in sys.argv else True, + time=True) + +import os + +# If ../mistral/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'mistral', '__init__.py')): + sys.path.insert(0, POSSIBLE_TOPDIR) + +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging as messaging +from wsgiref import simple_server + +from mistral.api import app +from terracotta import config +from terracotta import rpc +from terracotta.globals import manager as global_mgr + +from mistral import context as ctx +from mistral.db.v2 import api as db_api +from mistral.engine import default_engine as def_eng +from mistral.engine import default_executor as def_executor +from mistral.engine import rpc +from mistral.services import scheduler +from mistral import version + + +LOG = logging.getLogger(__name__) + + +def launch_executor(transport): + target = messaging.Target( + topic=cfg.CONF.executor.topic, + server=cfg.CONF.executor.host + ) + + executor_v2 = def_executor.DefaultExecutor(rpc.get_engine_client()) + + endpoints = [rpc.ExecutorServer(executor_v2)] + + server = messaging.get_rpc_server( + transport, + target, + endpoints, + executor='eventlet', + serializer=ctx.RpcContextSerializer(ctx.JsonPayloadSerializer()) + ) + + server.start() + server.wait() + + +def launch_engine(transport): + target = messaging.Target( + topic=cfg.CONF.engine.topic, + server=cfg.CONF.engine.host + ) + + engine_v2 = def_eng.DefaultEngine(rpc.get_engine_client()) + endpoints = [rpc.EngineServer(engine_v2)] + + + # Setup scheduler in engine. + db_api.setup_db() + scheduler.setup() + + server = messaging.get_rpc_server( + transport, + target, + endpoints, + executor='eventlet', + serializer=ctx.RpcContextSerializer(ctx.JsonPayloadSerializer()) + ) + + server.start() + server.wait() + + +def launch_gm(transport): + target = messaging.Target( + topic=cfg.CONF.global_manager.topic, + server=cfg.CONF.global_manager.host + ) + + engine_v2 = def_eng.DefaultEngine(rpc.get_engine_client()) + + endpoints = [rpc.EngineServer(engine_v2)] + + # Setup scheduler in engine. + db_api.setup_db() + scheduler.setup() + + server = messaging.get_rpc_server( + transport, + target, + endpoints, + executor='eventlet', + serializer=ctx.RpcContextSerializer(ctx.JsonPayloadSerializer()) + ) + + server.start() + server.wait() + + +def launch_api(transport): + host = cfg.CONF.api.host + port = cfg.CONF.api.port + + server = simple_server.make_server( + host, + port, + app.setup_app() + ) + + LOG.info("Mistral API is serving on http://%s:%s (PID=%s)" % + (host, port, os.getpid())) + + server.serve_forever() + + +def launch_any(transport, options): + # Launch the servers on different threads. + threads = [eventlet.spawn(LAUNCH_OPTIONS[option], transport) + for option in options] + + print('Server started.') + + [thread.wait() for thread in threads] + + +LAUNCH_OPTIONS = { + # 'api': launch_api, + 'global-manager': launch_gm, + 'local-collector': launch_collector, + 'local-manager': launch_lm +} + + +TERRACOTTA_TITLE = """ +##### ##### ##### ##### ##### ##### ##### ##### ##### ##### + # # # # # # # # # # # # # # # + # ##### ##### ##### ##### # # # # # ##### + # # # # # # # # # # # # # # # + # # # # # # # # # # # # # # # + # ##### # # # # # # ##### ##### # # # # + +Terracotta Dynamic Scheduling Service, version %s +""" % version.version_string() + + +def print_server_info(): + print(TERRACOTTA_TITLE) + + comp_str = ("[%s]" % ','.join(LAUNCH_OPTIONS) + if cfg.CONF.server == ['all'] else cfg.CONF.server) + + print('Launching server components %s...' % comp_str) + + +def main(): + try: + config.parse_args() + print_server_info() + logging.setup(cfg.CONF, 'Terracotta') + transport = rpc.get_transport() + + # Validate launch option. + if set(cfg.CONF.server) - set(LAUNCH_OPTIONS.keys()): + raise Exception('Valid options are all or any combination of ' + 'api, engine, and executor.') + + # Launch distinct set of server(s). + launch_any(transport, set(cfg.CONF.server)) + + except RuntimeError as excp: + sys.stderr.write("ERROR: %s\n" % excp) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/neat/common.py b/terracotta/common.py similarity index 100% rename from neat/common.py rename to terracotta/common.py diff --git a/terracotta/config.py b/terracotta/config.py new file mode 100644 index 0000000..9f4d46f --- /dev/null +++ b/terracotta/config.py @@ -0,0 +1,260 @@ +# Copyright 2012 Anton Beloglazov +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Configuration options registration and useful routines. +""" + +from oslo_config import cfg +from oslo_log import log as logging + +from terracotta import version + +from contracts import contract +import os +import ConfigParser + + +log = logging.getLogger(__name__) + + +# This is the default config, which should not be modified +DEFAULT_CONFIG_PATH = os.path.join(os.path.dirname(__file__), + '..', + 'neat.conf') + +# This is the custom config, which may override the defaults +CONFIG_PATH = "/etc/neat/neat.conf" +# The following value is used for testing purposes +#CONFIG_PATH = os.path.join(os.path.dirname(__file__), +# '..', +# 'neat.conf') + +# These fields must present in the configuration file +REQUIRED_FIELDS = [ + 'log_directory', + 'log_level', + 'vm_instance_directory', + 'sql_connection', + 'os_admin_tenant_name', + 'os_admin_user', + 'os_admin_password', + 'os_auth_url', + 'compute_hosts', + 'global_manager_host', + 'global_manager_port', + 'db_cleaner_interval', + 'local_data_directory', + 'local_manager_interval', + 'data_collector_interval', + 'data_collector_data_length', + 'host_cpu_overload_threshold', + 'host_cpu_usable_by_vms', + 'compute_user', + 'compute_password', + 'sleep_command', + 'ether_wake_interface', + 'block_migration', + 'network_migration_bandwidth', + 'algorithm_underload_detection_factory', + 'algorithm_underload_detection_parameters', + 'algorithm_overload_detection_factory', + 'algorithm_overload_detection_parameters', + 'algorithm_vm_selection_factory', + 'algorithm_vm_selection_parameters', + 'algorithm_vm_placement_factory', + 'algorithm_vm_placement_parameters', +] + + +@contract +def read_config(paths): + """ Read the configuration files and return the options. + + :param paths: A list of required configuration file paths. + :type paths: list(str) + + :return: A dictionary of the configuration options. + :rtype: dict(str: str) + """ + configParser = ConfigParser.ConfigParser() + for path in paths: + configParser.read(path) + return dict(configParser.items("DEFAULT")) + + +@contract +def validate_config(config, required_fields): + """ Check that the config contains all the required fields. + + :param config: A config dictionary to check. + :type config: dict(str: str) + + :param required_fields: A list of required fields. + :type required_fields: list(str) + + :return: Whether the config is valid. + :rtype: bool + """ + for field in required_fields: + if not field in config: + return False + return True + + +@contract +def read_and_validate_config(paths, required_fields): + """ Read the configuration files, validate and return the options. + + :param paths: A list of required configuration file paths. + :type paths: list(str) + + :param required_fields: A list of required fields. + :type required_fields: list(str) + + :return: A dictionary of the configuration options. + :rtype: dict(str: str) + """ + config = read_config(paths) + if not validate_config(config, required_fields): + message = 'The config dictionary does not contain ' + \ + 'all the required fields' + log.critical(message) + raise KeyError(message) + return config + + +launch_opt = cfg.ListOpt( + 'server', + default=['all'], + help='Specifies which mistral server to start by the launch script. ' + 'Valid options are all or any combination of ' + 'api, engine, and executor.' +) + +api_opts = [ + cfg.StrOpt('host', default='0.0.0.0', help='Mistral API server host'), + cfg.IntOpt('port', default=8989, help='Mistral API server port') +] + +pecan_opts = [ + cfg.StrOpt('root', default='mistral.api.controllers.root.RootController', + help='Pecan root controller'), + cfg.ListOpt('modules', default=["mistral.api"], + help='A list of modules where pecan will search for ' + 'applications.'), + cfg.BoolOpt('debug', default=False, + help='Enables the ability to display tracebacks in the ' + 'browser and interactively debug during ' + 'development.'), + cfg.BoolOpt('auth_enable', default=True, + help='Enables user authentication in pecan.') +] + +use_debugger = cfg.BoolOpt( + "use-debugger", + default=False, + help='Enables debugger. Note that using this option changes how the ' + 'eventlet library is used to support async IO. This could result ' + 'in failures that do not occur under normal operation. ' + 'Use at your own risk.' +) + +engine_opts = [ + cfg.StrOpt('engine', default='default', + help='Mistral engine plugin'), + cfg.StrOpt('host', default='0.0.0.0', + help='Name of the engine node. This can be an opaque ' + 'identifier. It is not necessarily a hostname, ' + 'FQDN, or IP address.'), + cfg.StrOpt('topic', default='engine', + help='The message topic that the engine listens on.'), + cfg.StrOpt('version', default='1.0', + help='The version of the engine.') +] + +executor_opts = [ + cfg.StrOpt('host', default='0.0.0.0', + help='Name of the executor node. This can be an opaque ' + 'identifier. It is not necessarily a hostname, ' + 'FQDN, or IP address.'), + cfg.StrOpt('topic', default='executor', + help='The message topic that the executor listens on.'), + cfg.StrOpt('version', default='1.0', + help='The version of the executor.') +] + +wf_trace_log_name_opt = cfg.StrOpt( + 'workflow_trace_log_name', + default='workflow_trace', + help='Logger name for pretty ' + 'workflow trace output.' +) + +CONF = cfg.CONF + +CONF.register_opts(api_opts, group='api') +CONF.register_opts(engine_opts, group='engine') +CONF.register_opts(pecan_opts, group='pecan') +CONF.register_opts(executor_opts, group='executor') +CONF.register_opt(wf_trace_log_name_opt) + +CONF.register_cli_opt(use_debugger) +CONF.register_cli_opt(launch_opt) + +CONF.import_opt('verbose', 'mistral.openstack.common.log') +CONF.set_default('verbose', True) +CONF.import_opt('debug', 'mistral.openstack.common.log') +CONF.import_opt('log_dir', 'mistral.openstack.common.log') +CONF.import_opt('log_file', 'mistral.openstack.common.log') +CONF.import_opt('log_config_append', 'mistral.openstack.common.log') +CONF.import_opt('log_format', 'mistral.openstack.common.log') +CONF.import_opt('log_date_format', 'mistral.openstack.common.log') +CONF.import_opt('use_syslog', 'mistral.openstack.common.log') +CONF.import_opt('syslog_log_facility', 'mistral.openstack.common.log') + +# Extend oslo default_log_levels to include some that are useful for mistral +# some are in oslo logging already, this is just making sure it stays this +# way. +default_log_levels = cfg.CONF.default_log_levels + +logs_to_quieten = [ + 'sqlalchemy=WARN', + 'oslo.messaging=INFO', + 'iso8601=WARN', + 'eventlet.wsgi.server=WARN', + 'stevedore=INFO', + 'mistral.openstack.common.loopingcall=INFO', + 'mistral.openstack.common.periodic_task=INFO', + 'mistral.services.periodic=INFO' +] + +for chatty in logs_to_quieten: + if chatty not in default_log_levels: + default_log_levels.append(chatty) + +cfg.set_defaults( + log.log_opts, + default_log_levels=default_log_levels +) + + +def parse_args(args=None, usage=None, default_config_files=None): + CONF( + args=args, + project='terracotta', + version=version, + usage=usage, + default_config_files=default_config_files + ) diff --git a/neat/contracts_extra.py b/terracotta/contracts_extra.py similarity index 100% rename from neat/contracts_extra.py rename to terracotta/contracts_extra.py diff --git a/neat/contracts_primitive.py b/terracotta/contracts_primitive.py similarity index 100% rename from neat/contracts_primitive.py rename to terracotta/contracts_primitive.py diff --git a/neat/db.py b/terracotta/db.py similarity index 100% rename from neat/db.py rename to terracotta/db.py diff --git a/neat/db_utils.py b/terracotta/db_utils.py similarity index 100% rename from neat/db_utils.py rename to terracotta/db_utils.py diff --git a/neat/locals/underload/__init__.py b/terracotta/globals/__init__.py similarity index 100% rename from neat/locals/underload/__init__.py rename to terracotta/globals/__init__.py diff --git a/neat/globals/db_cleaner.py b/terracotta/globals/db_cleaner.py similarity index 100% rename from neat/globals/db_cleaner.py rename to terracotta/globals/db_cleaner.py diff --git a/neat/globals/manager.py b/terracotta/globals/manager.py similarity index 100% rename from neat/globals/manager.py rename to terracotta/globals/manager.py diff --git a/neat/locals/vm_selection/__init__.py b/terracotta/globals/vm_placement/__init__.py similarity index 100% rename from neat/locals/vm_selection/__init__.py rename to terracotta/globals/vm_placement/__init__.py diff --git a/neat/globals/vm_placement/bin_packing.py b/terracotta/globals/vm_placement/bin_packing.py similarity index 100% rename from neat/globals/vm_placement/bin_packing.py rename to terracotta/globals/vm_placement/bin_packing.py diff --git a/terracotta/locals/__init__.py b/terracotta/locals/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/neat/locals/collector.py b/terracotta/locals/collector.py similarity index 100% rename from neat/locals/collector.py rename to terracotta/locals/collector.py diff --git a/neat/locals/manager.py b/terracotta/locals/manager.py similarity index 100% rename from neat/locals/manager.py rename to terracotta/locals/manager.py diff --git a/terracotta/locals/overload/__init__.py b/terracotta/locals/overload/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/terracotta/locals/overload/mhod/__init__.py b/terracotta/locals/overload/mhod/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/neat/locals/overload/mhod/bruteforce.py b/terracotta/locals/overload/mhod/bruteforce.py similarity index 100% rename from neat/locals/overload/mhod/bruteforce.py rename to terracotta/locals/overload/mhod/bruteforce.py diff --git a/neat/locals/overload/mhod/core.py b/terracotta/locals/overload/mhod/core.py similarity index 100% rename from neat/locals/overload/mhod/core.py rename to terracotta/locals/overload/mhod/core.py diff --git a/neat/locals/overload/mhod/l_2_states.py b/terracotta/locals/overload/mhod/l_2_states.py similarity index 100% rename from neat/locals/overload/mhod/l_2_states.py rename to terracotta/locals/overload/mhod/l_2_states.py diff --git a/neat/locals/overload/mhod/multisize_estimation.py b/terracotta/locals/overload/mhod/multisize_estimation.py similarity index 100% rename from neat/locals/overload/mhod/multisize_estimation.py rename to terracotta/locals/overload/mhod/multisize_estimation.py diff --git a/neat/locals/overload/mhod/nlp.py b/terracotta/locals/overload/mhod/nlp.py similarity index 100% rename from neat/locals/overload/mhod/nlp.py rename to terracotta/locals/overload/mhod/nlp.py diff --git a/neat/locals/overload/otf.py b/terracotta/locals/overload/otf.py similarity index 100% rename from neat/locals/overload/otf.py rename to terracotta/locals/overload/otf.py diff --git a/neat/locals/overload/statistics.py b/terracotta/locals/overload/statistics.py similarity index 100% rename from neat/locals/overload/statistics.py rename to terracotta/locals/overload/statistics.py diff --git a/neat/locals/overload/trivial.py b/terracotta/locals/overload/trivial.py similarity index 100% rename from neat/locals/overload/trivial.py rename to terracotta/locals/overload/trivial.py diff --git a/terracotta/locals/underload/__init__.py b/terracotta/locals/underload/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/neat/locals/underload/trivial.py b/terracotta/locals/underload/trivial.py similarity index 100% rename from neat/locals/underload/trivial.py rename to terracotta/locals/underload/trivial.py diff --git a/terracotta/locals/vm_selection/__init__.py b/terracotta/locals/vm_selection/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/neat/locals/vm_selection/algorithms.py b/terracotta/locals/vm_selection/algorithms.py similarity index 100% rename from neat/locals/vm_selection/algorithms.py rename to terracotta/locals/vm_selection/algorithms.py diff --git a/terracotta/openstack/__init__.py b/terracotta/openstack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/terracotta/openstack/common/README b/terracotta/openstack/common/README new file mode 100644 index 0000000..04a6166 --- /dev/null +++ b/terracotta/openstack/common/README @@ -0,0 +1,16 @@ +oslo-incubator +-------------- + +A number of modules from oslo-incubator are imported into this project. +You can clone the oslo-incubator repository using the following url: + + git://git.openstack.org/openstack/oslo-incubator + +These modules are "incubating" in oslo-incubator and are kept in sync +with the help of oslo-incubator's update.py script. See: + + https://wiki.openstack.org/wiki/Oslo#Syncing_Code_from_Incubator + +The copy of the code should never be directly modified here. Please +always update oslo-incubator first and then run the script to copy +the changes across. diff --git a/terracotta/openstack/common/__init__.py b/terracotta/openstack/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/terracotta/openstack/common/_i18n.py b/terracotta/openstack/common/_i18n.py new file mode 100644 index 0000000..391f321 --- /dev/null +++ b/terracotta/openstack/common/_i18n.py @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""oslo.i18n integration module. + +See http://docs.openstack.org/developer/oslo.i18n/usage.html + +""" + +try: + import oslo_i18n + + # NOTE(dhellmann): This reference to o-s-l-o will be replaced by the + # application name when this module is synced into the separate + # repository. It is OK to have more than one translation function + # using the same domain, since there will still only be one message + # catalog. + _translators = oslo_i18n.TranslatorFactory(domain='nova') + + # The primary translation function using the well-known name "_" + _ = _translators.primary + + # Translators for log levels. + # + # The abbreviated names are meant to reflect the usual use of a short + # name like '_'. The "L" is for "log" and the other letter comes from + # the level. + _LI = _translators.log_info + _LW = _translators.log_warning + _LE = _translators.log_error + _LC = _translators.log_critical +except ImportError: + # NOTE(dims): Support for cases where a project wants to use + # code from oslo-incubator, but is not ready to be internationalized + # (like tempest) + _ = _LI = _LW = _LE = _LC = lambda x: x diff --git a/terracotta/openstack/common/cliutils.py b/terracotta/openstack/common/cliutils.py new file mode 100644 index 0000000..33b475e --- /dev/null +++ b/terracotta/openstack/common/cliutils.py @@ -0,0 +1,271 @@ +# Copyright 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# W0603: Using the global statement +# W0621: Redefining name %s from outer scope +# pylint: disable=W0603,W0621 + +from __future__ import print_function + +import getpass +import inspect +import os +import sys +import textwrap + +from oslo_utils import encodeutils +from oslo_utils import strutils +import prettytable +import six +from six import moves + +from nova.openstack.common._i18n import _ + + +class MissingArgs(Exception): + """Supplied arguments are not sufficient for calling a function.""" + def __init__(self, missing): + self.missing = missing + msg = _("Missing arguments: %s") % ", ".join(missing) + super(MissingArgs, self).__init__(msg) + + +def validate_args(fn, *args, **kwargs): + """Check that the supplied args are sufficient for calling a function. + + >>> validate_args(lambda a: None) + Traceback (most recent call last): + ... + MissingArgs: Missing argument(s): a + >>> validate_args(lambda a, b, c, d: None, 0, c=1) + Traceback (most recent call last): + ... + MissingArgs: Missing argument(s): b, d + + :param fn: the function to check + :param arg: the positional arguments supplied + :param kwargs: the keyword arguments supplied + """ + argspec = inspect.getargspec(fn) + + num_defaults = len(argspec.defaults or []) + required_args = argspec.args[:len(argspec.args) - num_defaults] + + def isbound(method): + return getattr(method, '__self__', None) is not None + + if isbound(fn): + required_args.pop(0) + + missing = [arg for arg in required_args if arg not in kwargs] + missing = missing[len(args):] + if missing: + raise MissingArgs(missing) + + +def arg(*args, **kwargs): + """Decorator for CLI args. + + Example: + + >>> @arg("name", help="Name of the new entity") + ... def entity_create(args): + ... pass + """ + def _decorator(func): + add_arg(func, *args, **kwargs) + return func + return _decorator + + +def env(*args, **kwargs): + """Returns the first environment variable set. + + If all are empty, defaults to '' or keyword arg `default`. + """ + for arg in args: + value = os.environ.get(arg) + if value: + return value + return kwargs.get('default', '') + + +def add_arg(func, *args, **kwargs): + """Bind CLI arguments to a shell.py `do_foo` function.""" + + if not hasattr(func, 'arguments'): + func.arguments = [] + + # NOTE(sirp): avoid dups that can occur when the module is shared across + # tests. + if (args, kwargs) not in func.arguments: + # Because of the semantics of decorator composition if we just append + # to the options list positional options will appear to be backwards. + func.arguments.insert(0, (args, kwargs)) + + +def unauthenticated(func): + """Adds 'unauthenticated' attribute to decorated function. + + Usage: + + >>> @unauthenticated + ... def mymethod(f): + ... pass + """ + func.unauthenticated = True + return func + + +def isunauthenticated(func): + """Checks if the function does not require authentication. + + Mark such functions with the `@unauthenticated` decorator. + + :returns: bool + """ + return getattr(func, 'unauthenticated', False) + + +def print_list(objs, fields, formatters=None, sortby_index=0, + mixed_case_fields=None, field_labels=None): + """Print a list or objects as a table, one row per object. + + :param objs: iterable of :class:`Resource` + :param fields: attributes that correspond to columns, in order + :param formatters: `dict` of callables for field formatting + :param sortby_index: index of the field for sorting table rows + :param mixed_case_fields: fields corresponding to object attributes that + have mixed case names (e.g., 'serverId') + :param field_labels: Labels to use in the heading of the table, default to + fields. + """ + formatters = formatters or {} + mixed_case_fields = mixed_case_fields or [] + field_labels = field_labels or fields + if len(field_labels) != len(fields): + raise ValueError(_("Field labels list %(labels)s has different number " + "of elements than fields list %(fields)s"), + {'labels': field_labels, 'fields': fields}) + + if sortby_index is None: + kwargs = {} + else: + kwargs = {'sortby': field_labels[sortby_index]} + pt = prettytable.PrettyTable(field_labels) + pt.align = 'l' + + for o in objs: + row = [] + for field in fields: + if field in formatters: + row.append(formatters[field](o)) + else: + if field in mixed_case_fields: + field_name = field.replace(' ', '_') + else: + field_name = field.lower().replace(' ', '_') + data = getattr(o, field_name, '') + row.append(data) + pt.add_row(row) + + if six.PY3: + print(encodeutils.safe_encode(pt.get_string(**kwargs)).decode()) + else: + print(encodeutils.safe_encode(pt.get_string(**kwargs))) + + +def print_dict(dct, dict_property="Property", wrap=0): + """Print a `dict` as a table of two columns. + + :param dct: `dict` to print + :param dict_property: name of the first column + :param wrap: wrapping for the second column + """ + pt = prettytable.PrettyTable([dict_property, 'Value']) + pt.align = 'l' + for k, v in six.iteritems(dct): + # convert dict to str to check length + if isinstance(v, dict): + v = six.text_type(v) + if wrap > 0: + v = textwrap.fill(six.text_type(v), wrap) + # if value has a newline, add in multiple rows + # e.g. fault with stacktrace + if v and isinstance(v, six.string_types) and r'\n' in v: + lines = v.strip().split(r'\n') + col1 = k + for line in lines: + pt.add_row([col1, line]) + col1 = '' + else: + pt.add_row([k, v]) + + if six.PY3: + print(encodeutils.safe_encode(pt.get_string()).decode()) + else: + print(encodeutils.safe_encode(pt.get_string())) + + +def get_password(max_password_prompts=3): + """Read password from TTY.""" + verify = strutils.bool_from_string(env("OS_VERIFY_PASSWORD")) + pw = None + if hasattr(sys.stdin, "isatty") and sys.stdin.isatty(): + # Check for Ctrl-D + try: + for __ in moves.range(max_password_prompts): + pw1 = getpass.getpass("OS Password: ") + if verify: + pw2 = getpass.getpass("Please verify: ") + else: + pw2 = pw1 + if pw1 == pw2 and pw1: + pw = pw1 + break + except EOFError: + pass + return pw + + +def service_type(stype): + """Adds 'service_type' attribute to decorated function. + + Usage: + + .. code-block:: python + + @service_type('volume') + def mymethod(f): + ... + """ + def inner(f): + f.service_type = stype + return f + return inner + + +def get_service_type(f): + """Retrieves service type from function.""" + return getattr(f, 'service_type', None) + + +def pretty_choice_list(l): + return ', '.join("'%s'" % i for i in l) + + +def exit(msg=''): + if msg: + print (msg, file=sys.stderr) + sys.exit(1) diff --git a/terracotta/openstack/common/config/__init__.py b/terracotta/openstack/common/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/terracotta/openstack/common/config/generator.py b/terracotta/openstack/common/config/generator.py new file mode 100644 index 0000000..a4b2ab5 --- /dev/null +++ b/terracotta/openstack/common/config/generator.py @@ -0,0 +1,320 @@ +# Copyright 2012 SINA Corporation +# Copyright 2014 Cisco Systems, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""Extracts OpenStack config option info from module(s).""" + +from __future__ import print_function + +import argparse +import imp +import os +import re +import socket +import sys +import textwrap + +from oslo_config import cfg +from oslo_utils import importutils +import six +import stevedore.named + +STROPT = "StrOpt" +BOOLOPT = "BoolOpt" +INTOPT = "IntOpt" +FLOATOPT = "FloatOpt" +LISTOPT = "ListOpt" +DICTOPT = "DictOpt" +MULTISTROPT = "MultiStrOpt" + +OPT_TYPES = { + STROPT: 'string value', + BOOLOPT: 'boolean value', + INTOPT: 'integer value', + FLOATOPT: 'floating point value', + LISTOPT: 'list value', + DICTOPT: 'dict value', + MULTISTROPT: 'multi valued', +} + +OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT, + FLOATOPT, LISTOPT, DICTOPT, + MULTISTROPT])) + +PY_EXT = ".py" +BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), + "../../../../")) +WORDWRAP_WIDTH = 60 + + +def raise_extension_exception(extmanager, ep, err): + raise + + +def generate(argv): + parser = argparse.ArgumentParser( + description='generate sample configuration file', + ) + parser.add_argument('-m', dest='modules', action='append') + parser.add_argument('-l', dest='libraries', action='append') + parser.add_argument('srcfiles', nargs='*') + parsed_args = parser.parse_args(argv) + + mods_by_pkg = dict() + for filepath in parsed_args.srcfiles: + pkg_name = filepath.split(os.sep)[1] + mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]), + os.path.basename(filepath).split('.')[0]]) + mods_by_pkg.setdefault(pkg_name, list()).append(mod_str) + # NOTE(lzyeval): place top level modules before packages + pkg_names = sorted(pkg for pkg in mods_by_pkg if pkg.endswith(PY_EXT)) + ext_names = sorted(pkg for pkg in mods_by_pkg if pkg not in pkg_names) + pkg_names.extend(ext_names) + + # opts_by_group is a mapping of group name to an options list + # The options list is a list of (module, options) tuples + opts_by_group = {'DEFAULT': []} + + if parsed_args.modules: + for module_name in parsed_args.modules: + module = _import_module(module_name) + if module: + for group, opts in _list_opts(module): + opts_by_group.setdefault(group, []).append((module_name, + opts)) + + # Look for entry points defined in libraries (or applications) for + # option discovery, and include their return values in the output. + # + # Each entry point should be a function returning an iterable + # of pairs with the group name (or None for the default group) + # and the list of Opt instances for that group. + if parsed_args.libraries: + loader = stevedore.named.NamedExtensionManager( + 'oslo_config.opts', + names=list(set(parsed_args.libraries)), + invoke_on_load=False, + on_load_failure_callback=raise_extension_exception + ) + for ext in loader: + for group, opts in ext.plugin(): + opt_list = opts_by_group.setdefault(group or 'DEFAULT', []) + opt_list.append((ext.name, opts)) + + for pkg_name in pkg_names: + mods = mods_by_pkg.get(pkg_name) + mods.sort() + for mod_str in mods: + if mod_str.endswith('.__init__'): + mod_str = mod_str[:mod_str.rfind(".")] + + mod_obj = _import_module(mod_str) + if not mod_obj: + raise RuntimeError("Unable to import module %s" % mod_str) + + for group, opts in _list_opts(mod_obj): + opts_by_group.setdefault(group, []).append((mod_str, opts)) + + print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', [])) + for group in sorted(opts_by_group.keys()): + print_group_opts(group, opts_by_group[group]) + + +def _import_module(mod_str): + try: + if mod_str.startswith('bin.'): + imp.load_source(mod_str[4:], os.path.join('bin', mod_str[4:])) + return sys.modules[mod_str[4:]] + else: + return importutils.import_module(mod_str) + except Exception as e: + sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e))) + return None + + +def _is_in_group(opt, group): + "Check if opt is in group." + for value in group._opts.values(): + # NOTE(llu): Temporary workaround for bug #1262148, wait until + # newly released oslo.config support '==' operator. + if not(value['opt'] != opt): + return True + return False + + +def _guess_groups(opt, mod_obj): + # is it in the DEFAULT group? + if _is_in_group(opt, cfg.CONF): + return 'DEFAULT' + + # what other groups is it in? + for value in cfg.CONF.values(): + if isinstance(value, cfg.CONF.GroupAttr): + if _is_in_group(opt, value._group): + return value._group.name + + raise RuntimeError( + "Unable to find group for option %s, " + "maybe it's defined twice in the same group?" + % opt.name + ) + + +def _list_opts(obj): + def is_opt(o): + return (isinstance(o, cfg.Opt) and + not isinstance(o, cfg.SubCommandOpt)) + + opts = list() + + if 'list_opts' in dir(obj): + return getattr(obj, 'list_opts')() + + for attr_str in dir(obj): + attr_obj = getattr(obj, attr_str) + if is_opt(attr_obj): + opts.append(attr_obj) + elif (isinstance(attr_obj, list) and + all(map(lambda x: is_opt(x), attr_obj))): + opts.extend(attr_obj) + + ret = {} + for opt in opts: + ret.setdefault(_guess_groups(opt, obj), []).append(opt) + return ret.items() + + +def print_group_opts(group, opts_by_module): + print("[%s]" % group) + print('') + for mod, opts in opts_by_module: + print('#') + print('# Options defined in %s' % mod) + print('#') + print('') + for opt in opts: + _print_opt(opt) + print('') + + +def _get_my_ip(): + try: + csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + csock.connect(('8.8.8.8', 80)) + (addr, port) = csock.getsockname() + csock.close() + return addr + except socket.error: + return None + + +def _sanitize_default(name, value): + """Set up a reasonably sensible default for pybasedir, my_ip and host.""" + if value.startswith(sys.prefix): + # NOTE(jd) Don't use os.path.join, because it is likely to think the + # second part is an absolute pathname and therefore drop the first + # part. + value = os.path.normpath("/usr/" + value[len(sys.prefix):]) + elif value.startswith(BASEDIR): + return value.replace(BASEDIR, '/usr/lib/python/site-packages') + elif BASEDIR in value: + return value.replace(BASEDIR, '') + elif value == _get_my_ip(): + return '10.0.0.1' + elif value in (socket.gethostname(), socket.getfqdn()) and 'host' in name: + return 'nova' + elif value.strip() != value: + return '"%s"' % value + return value + + +def _get_choice_text(choice): + if choice is None: + return '' + elif choice == '': + return "''" + return six.text_type(choice) + + +def _print_opt(opt): + opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help + if not opt_help: + sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name) + opt_help = "" + opt_type = None + try: + opt_type = OPTION_REGEX.search(str(type(opt))).group(0) + except (ValueError, AttributeError) as err: + sys.stderr.write("%s\n" % str(err)) + sys.exit(1) + opt_help = u'%s (%s)' % (opt_help, + OPT_TYPES[opt_type]) + print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))) + if opt.deprecated_opts: + for deprecated_opt in opt.deprecated_opts: + if deprecated_opt.name: + deprecated_group = (deprecated_opt.group if + deprecated_opt.group else "DEFAULT") + print('# Deprecated group/name - [%s]/%s' % + (deprecated_group, + deprecated_opt.name)) + try: + if opt_default is None: + print('#%s=' % opt_name) + elif opt_type == STROPT: + assert(isinstance(opt_default, six.string_types)) + if (getattr(opt, 'type', None) and + getattr(opt.type, 'choices', None)): + choices_text = ', '.join([_get_choice_text(choice) + for choice in opt.type.choices]) + print('# Allowed values: %s' % choices_text) + print('#%s=%s' % (opt_name, _sanitize_default(opt_name, + opt_default))) + elif opt_type == BOOLOPT: + assert(isinstance(opt_default, bool)) + print('#%s=%s' % (opt_name, str(opt_default).lower())) + elif opt_type == INTOPT: + assert(isinstance(opt_default, int) and + not isinstance(opt_default, bool)) + print('#%s=%s' % (opt_name, opt_default)) + elif opt_type == FLOATOPT: + assert(isinstance(opt_default, float)) + print('#%s=%s' % (opt_name, opt_default)) + elif opt_type == LISTOPT: + assert(isinstance(opt_default, list)) + print('#%s=%s' % (opt_name, ','.join(opt_default))) + elif opt_type == DICTOPT: + assert(isinstance(opt_default, dict)) + opt_default_strlist = [str(key) + ':' + str(value) + for (key, value) in opt_default.items()] + print('#%s=%s' % (opt_name, ','.join(opt_default_strlist))) + elif opt_type == MULTISTROPT: + assert(isinstance(opt_default, list)) + if not opt_default: + opt_default = [''] + for default in opt_default: + print('#%s=%s' % (opt_name, default)) + print('') + except Exception: + sys.stderr.write('Error in option "%s"\n' % opt_name) + sys.exit(1) + + +def main(): + generate(sys.argv[1:]) + +if __name__ == '__main__': + main() diff --git a/terracotta/openstack/common/eventlet_backdoor.py b/terracotta/openstack/common/eventlet_backdoor.py new file mode 100644 index 0000000..2b9bf50 --- /dev/null +++ b/terracotta/openstack/common/eventlet_backdoor.py @@ -0,0 +1,151 @@ +# Copyright (c) 2012 OpenStack Foundation. +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import copy +import errno +import gc +import logging +import os +import pprint +import socket +import sys +import traceback + +import eventlet.backdoor +import greenlet +from oslo_config import cfg + +from nova.openstack.common._i18n import _LI + +help_for_backdoor_port = ( + "Acceptable values are 0, , and :, where 0 results " + "in listening on a random tcp port number; results in listening " + "on the specified port number (and not enabling backdoor if that port " + "is in use); and : results in listening on the smallest " + "unused port number within the specified range of port numbers. The " + "chosen port is displayed in the service's log file.") +eventlet_backdoor_opts = [ + cfg.StrOpt('backdoor_port', + help="Enable eventlet backdoor. %s" % help_for_backdoor_port) +] + +CONF = cfg.CONF +CONF.register_opts(eventlet_backdoor_opts) +LOG = logging.getLogger(__name__) + + +def list_opts(): + """Entry point for oslo-config-generator. + """ + return [(None, copy.deepcopy(eventlet_backdoor_opts))] + + +class EventletBackdoorConfigValueError(Exception): + def __init__(self, port_range, help_msg, ex): + msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. ' + '%(help)s' % + {'range': port_range, 'ex': ex, 'help': help_msg}) + super(EventletBackdoorConfigValueError, self).__init__(msg) + self.port_range = port_range + + +def _dont_use_this(): + print("Don't use this, just disconnect instead") + + +def _find_objects(t): + return [o for o in gc.get_objects() if isinstance(o, t)] + + +def _print_greenthreads(): + for i, gt in enumerate(_find_objects(greenlet.greenlet)): + print(i, gt) + traceback.print_stack(gt.gr_frame) + print() + + +def _print_nativethreads(): + for threadId, stack in sys._current_frames().items(): + print(threadId) + traceback.print_stack(stack) + print() + + +def _parse_port_range(port_range): + if ':' not in port_range: + start, end = port_range, port_range + else: + start, end = port_range.split(':', 1) + try: + start, end = int(start), int(end) + if end < start: + raise ValueError + return start, end + except ValueError as ex: + raise EventletBackdoorConfigValueError(port_range, ex, + help_for_backdoor_port) + + +def _listen(host, start_port, end_port, listen_func): + try_port = start_port + while True: + try: + return listen_func((host, try_port)) + except socket.error as exc: + if (exc.errno != errno.EADDRINUSE or + try_port >= end_port): + raise + try_port += 1 + + +def initialize_if_enabled(): + backdoor_locals = { + 'exit': _dont_use_this, # So we don't exit the entire process + 'quit': _dont_use_this, # So we don't exit the entire process + 'fo': _find_objects, + 'pgt': _print_greenthreads, + 'pnt': _print_nativethreads, + } + + if CONF.backdoor_port is None: + return None + + start_port, end_port = _parse_port_range(str(CONF.backdoor_port)) + + # NOTE(johannes): The standard sys.displayhook will print the value of + # the last expression and set it to __builtin__._, which overwrites + # the __builtin__._ that gettext sets. Let's switch to using pprint + # since it won't interact poorly with gettext, and it's easier to + # read the output too. + def displayhook(val): + if val is not None: + pprint.pprint(val) + sys.displayhook = displayhook + + sock = _listen('localhost', start_port, end_port, eventlet.listen) + + # In the case of backdoor port being zero, a port number is assigned by + # listen(). In any case, pull the port number out here. + port = sock.getsockname()[1] + LOG.info( + _LI('Eventlet backdoor listening on %(port)s for process %(pid)d') % + {'port': port, 'pid': os.getpid()} + ) + eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock, + locals=backdoor_locals) + return port diff --git a/terracotta/openstack/common/fileutils.py b/terracotta/openstack/common/fileutils.py new file mode 100644 index 0000000..9097c35 --- /dev/null +++ b/terracotta/openstack/common/fileutils.py @@ -0,0 +1,149 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib +import errno +import logging +import os +import stat +import tempfile + +from oslo_utils import excutils + +LOG = logging.getLogger(__name__) + +_FILE_CACHE = {} +DEFAULT_MODE = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + + +def ensure_tree(path, mode=DEFAULT_MODE): + """Create a directory (and any ancestor directories required) + + :param path: Directory to create + :param mode: Directory creation permissions + """ + try: + os.makedirs(path, mode) + except OSError as exc: + if exc.errno == errno.EEXIST: + if not os.path.isdir(path): + raise + else: + raise + + +def read_cached_file(filename, force_reload=False): + """Read from a file if it has been modified. + + :param force_reload: Whether to reload the file. + :returns: A tuple with a boolean specifying if the data is fresh + or not. + """ + global _FILE_CACHE + + if force_reload: + delete_cached_file(filename) + + reloaded = False + mtime = os.path.getmtime(filename) + cache_info = _FILE_CACHE.setdefault(filename, {}) + + if not cache_info or mtime > cache_info.get('mtime', 0): + LOG.debug("Reloading cached file %s" % filename) + with open(filename) as fap: + cache_info['data'] = fap.read() + cache_info['mtime'] = mtime + reloaded = True + return (reloaded, cache_info['data']) + + +def delete_cached_file(filename): + """Delete cached file if present. + + :param filename: filename to delete + """ + global _FILE_CACHE + + if filename in _FILE_CACHE: + del _FILE_CACHE[filename] + + +def delete_if_exists(path, remove=os.unlink): + """Delete a file, but ignore file not found error. + + :param path: File to delete + :param remove: Optional function to remove passed path + """ + + try: + remove(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + +@contextlib.contextmanager +def remove_path_on_error(path, remove=delete_if_exists): + """Protect code that wants to operate on PATH atomically. + Any exception will cause PATH to be removed. + + :param path: File to work with + :param remove: Optional function to remove passed path + """ + + try: + yield + except Exception: + with excutils.save_and_reraise_exception(): + remove(path) + + +def file_open(*args, **kwargs): + """Open file + + see built-in open() documentation for more details + + Note: The reason this is kept in a separate module is to easily + be able to provide a stub module that doesn't alter system + state at all (for unit tests) + """ + return open(*args, **kwargs) + + +def write_to_tempfile(content, path=None, suffix='', prefix='tmp'): + """Create temporary file or use existing file. + + This util is needed for creating temporary file with + specified content, suffix and prefix. If path is not None, + it will be used for writing content. If the path doesn't + exist it'll be created. + + :param content: content for temporary file. + :param path: same as parameter 'dir' for mkstemp + :param suffix: same as parameter 'suffix' for mkstemp + :param prefix: same as parameter 'prefix' for mkstemp + + For example: it can be used in database tests for creating + configuration files. + """ + if path: + ensure_tree(path) + + (fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix) + try: + os.write(fd, content) + finally: + os.close(fd) + return path diff --git a/terracotta/openstack/common/imageutils.py b/terracotta/openstack/common/imageutils.py new file mode 100644 index 0000000..b37ce24 --- /dev/null +++ b/terracotta/openstack/common/imageutils.py @@ -0,0 +1,152 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Helper methods to deal with images. +""" + +import re + +from oslo_utils import strutils + +from nova.openstack.common._i18n import _ + + +class QemuImgInfo(object): + BACKING_FILE_RE = re.compile((r"^(.*?)\s*\(actual\s+path\s*:" + r"\s+(.*?)\)\s*$"), re.I) + TOP_LEVEL_RE = re.compile(r"^([\w\d\s\_\-]+):(.*)$") + SIZE_RE = re.compile(r"(\d*\.?\d+)(\w+)?(\s*\(\s*(\d+)\s+bytes\s*\))?", + re.I) + + def __init__(self, cmd_output=None): + details = self._parse(cmd_output or '') + self.image = details.get('image') + self.backing_file = details.get('backing_file') + self.file_format = details.get('file_format') + self.virtual_size = details.get('virtual_size') + self.cluster_size = details.get('cluster_size') + self.disk_size = details.get('disk_size') + self.snapshots = details.get('snapshot_list', []) + self.encrypted = details.get('encrypted') + + def __str__(self): + lines = [ + 'image: %s' % self.image, + 'file_format: %s' % self.file_format, + 'virtual_size: %s' % self.virtual_size, + 'disk_size: %s' % self.disk_size, + 'cluster_size: %s' % self.cluster_size, + 'backing_file: %s' % self.backing_file, + ] + if self.snapshots: + lines.append("snapshots: %s" % self.snapshots) + if self.encrypted: + lines.append("encrypted: %s" % self.encrypted) + return "\n".join(lines) + + def _canonicalize(self, field): + # Standardize on underscores/lc/no dash and no spaces + # since qemu seems to have mixed outputs here... and + # this format allows for better integration with python + # - i.e. for usage in kwargs and such... + field = field.lower().strip() + for c in (" ", "-"): + field = field.replace(c, '_') + return field + + def _extract_bytes(self, details): + # Replace it with the byte amount + real_size = self.SIZE_RE.search(details) + if not real_size: + raise ValueError(_('Invalid input value "%s".') % details) + magnitude = real_size.group(1) + unit_of_measure = real_size.group(2) + bytes_info = real_size.group(3) + if bytes_info: + return int(real_size.group(4)) + elif not unit_of_measure: + return int(magnitude) + return strutils.string_to_bytes('%s%sB' % (magnitude, unit_of_measure), + return_int=True) + + def _extract_details(self, root_cmd, root_details, lines_after): + real_details = root_details + if root_cmd == 'backing_file': + # Replace it with the real backing file + backing_match = self.BACKING_FILE_RE.match(root_details) + if backing_match: + real_details = backing_match.group(2).strip() + elif root_cmd in ['virtual_size', 'cluster_size', 'disk_size']: + # Replace it with the byte amount (if we can convert it) + if root_details == 'None': + real_details = 0 + else: + real_details = self._extract_bytes(root_details) + elif root_cmd == 'file_format': + real_details = real_details.strip().lower() + elif root_cmd == 'snapshot_list': + # Next line should be a header, starting with 'ID' + if not lines_after or not lines_after.pop(0).startswith("ID"): + msg = _("Snapshot list encountered but no header found!") + raise ValueError(msg) + real_details = [] + # This is the sprintf pattern we will try to match + # "%-10s%-20s%7s%20s%15s" + # ID TAG VM SIZE DATE VM CLOCK (current header) + while lines_after: + line = lines_after[0] + line_pieces = line.split() + if len(line_pieces) != 6: + break + # Check against this pattern in the final position + # "%02d:%02d:%02d.%03d" + date_pieces = line_pieces[5].split(":") + if len(date_pieces) != 3: + break + lines_after.pop(0) + real_details.append({ + 'id': line_pieces[0], + 'tag': line_pieces[1], + 'vm_size': line_pieces[2], + 'date': line_pieces[3], + 'vm_clock': line_pieces[4] + " " + line_pieces[5], + }) + return real_details + + def _parse(self, cmd_output): + # Analysis done of qemu-img.c to figure out what is going on here + # Find all points start with some chars and then a ':' then a newline + # and then handle the results of those 'top level' items in a separate + # function. + # + # TODO(harlowja): newer versions might have a json output format + # we should switch to that whenever possible. + # see: http://bit.ly/XLJXDX + contents = {} + lines = [x for x in cmd_output.splitlines() if x.strip()] + while lines: + line = lines.pop(0) + top_level = self.TOP_LEVEL_RE.match(line) + if top_level: + root = self._canonicalize(top_level.group(1)) + if not root: + continue + root_details = top_level.group(2).strip() + details = self._extract_details(root, root_details, lines) + contents[root] = details + return contents diff --git a/terracotta/openstack/common/local.py b/terracotta/openstack/common/local.py new file mode 100644 index 0000000..0819d5b --- /dev/null +++ b/terracotta/openstack/common/local.py @@ -0,0 +1,45 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Local storage of variables using weak references""" + +import threading +import weakref + + +class WeakLocal(threading.local): + def __getattribute__(self, attr): + rval = super(WeakLocal, self).__getattribute__(attr) + if rval: + # NOTE(mikal): this bit is confusing. What is stored is a weak + # reference, not the value itself. We therefore need to lookup + # the weak reference and return the inner value here. + rval = rval() + return rval + + def __setattr__(self, attr, value): + value = weakref.ref(value) + return super(WeakLocal, self).__setattr__(attr, value) + + +# NOTE(mikal): the name "store" should be deprecated in the future +store = WeakLocal() + +# A "weak" store uses weak references and allows an object to fall out of scope +# when it falls out of scope in the code that uses the thread local storage. A +# "strong" store will hold a reference to the object so that it never falls out +# of scope. +weak_store = WeakLocal() +strong_store = threading.local() diff --git a/terracotta/openstack/common/loopingcall.py b/terracotta/openstack/common/loopingcall.py new file mode 100644 index 0000000..29628b7 --- /dev/null +++ b/terracotta/openstack/common/loopingcall.py @@ -0,0 +1,147 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys +import time + +from eventlet import event +from eventlet import greenthread + +from nova.openstack.common._i18n import _LE, _LW + +LOG = logging.getLogger(__name__) + +# NOTE(zyluo): This lambda function was declared to avoid mocking collisions +# with time.time() called in the standard logging module +# during unittests. +_ts = lambda: time.time() + + +class LoopingCallDone(Exception): + """Exception to break out and stop a LoopingCallBase. + + The poll-function passed to LoopingCallBase can raise this exception to + break out of the loop normally. This is somewhat analogous to + StopIteration. + + An optional return-value can be included as the argument to the exception; + this return-value will be returned by LoopingCallBase.wait() + + """ + + def __init__(self, retvalue=True): + """:param retvalue: Value that LoopingCallBase.wait() should return.""" + self.retvalue = retvalue + + +class LoopingCallBase(object): + def __init__(self, f=None, *args, **kw): + self.args = args + self.kw = kw + self.f = f + self._running = False + self.done = None + + def stop(self): + self._running = False + + def wait(self): + return self.done.wait() + + +class FixedIntervalLoopingCall(LoopingCallBase): + """A fixed interval looping call.""" + + def start(self, interval, initial_delay=None): + self._running = True + done = event.Event() + + def _inner(): + if initial_delay: + greenthread.sleep(initial_delay) + + try: + while self._running: + start = _ts() + self.f(*self.args, **self.kw) + end = _ts() + if not self._running: + break + delay = end - start - interval + if delay > 0: + LOG.warn(_LW('task %(func_name)r run outlasted ' + 'interval by %(delay).2f sec'), + {'func_name': self.f, 'delay': delay}) + greenthread.sleep(-delay if delay < 0 else 0) + except LoopingCallDone as e: + self.stop() + done.send(e.retvalue) + except Exception: + LOG.exception(_LE('in fixed duration looping call')) + done.send_exception(*sys.exc_info()) + return + else: + done.send(True) + + self.done = done + + greenthread.spawn_n(_inner) + return self.done + + +class DynamicLoopingCall(LoopingCallBase): + """A looping call which sleeps until the next known event. + + The function called should return how long to sleep for before being + called again. + """ + + def start(self, initial_delay=None, periodic_interval_max=None): + self._running = True + done = event.Event() + + def _inner(): + if initial_delay: + greenthread.sleep(initial_delay) + + try: + while self._running: + idle = self.f(*self.args, **self.kw) + if not self._running: + break + + if periodic_interval_max is not None: + idle = min(idle, periodic_interval_max) + LOG.debug('Dynamic looping call %(func_name)r sleeping ' + 'for %(idle).02f seconds', + {'func_name': self.f, 'idle': idle}) + greenthread.sleep(idle) + except LoopingCallDone as e: + self.stop() + done.send(e.retvalue) + except Exception: + LOG.exception(_LE('in dynamic looping call')) + done.send_exception(*sys.exc_info()) + return + else: + done.send(True) + + self.done = done + + greenthread.spawn(_inner) + return self.done diff --git a/terracotta/openstack/common/memorycache.py b/terracotta/openstack/common/memorycache.py new file mode 100644 index 0000000..e72c26d --- /dev/null +++ b/terracotta/openstack/common/memorycache.py @@ -0,0 +1,97 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Super simple fake memcache client.""" + +import copy + +from oslo_config import cfg +from oslo_utils import timeutils + +memcache_opts = [ + cfg.ListOpt('memcached_servers', + help='Memcached servers or None for in process cache.'), +] + +CONF = cfg.CONF +CONF.register_opts(memcache_opts) + + +def list_opts(): + """Entry point for oslo-config-generator.""" + return [(None, copy.deepcopy(memcache_opts))] + + +def get_client(memcached_servers=None): + client_cls = Client + + if not memcached_servers: + memcached_servers = CONF.memcached_servers + if memcached_servers: + import memcache + client_cls = memcache.Client + + return client_cls(memcached_servers, debug=0) + + +class Client(object): + """Replicates a tiny subset of memcached client interface.""" + + def __init__(self, *args, **kwargs): + """Ignores the passed in args.""" + self.cache = {} + + def get(self, key): + """Retrieves the value for a key or None. + + This expunges expired keys during each get. + """ + + now = timeutils.utcnow_ts() + for k in list(self.cache): + (timeout, _value) = self.cache[k] + if timeout and now >= timeout: + del self.cache[k] + + return self.cache.get(key, (0, None))[1] + + def set(self, key, value, time=0, min_compress_len=0): + """Sets the value for a key.""" + timeout = 0 + if time != 0: + timeout = timeutils.utcnow_ts() + time + self.cache[key] = (timeout, value) + return True + + def add(self, key, value, time=0, min_compress_len=0): + """Sets the value for a key if it doesn't exist.""" + if self.get(key) is not None: + return False + return self.set(key, value, time, min_compress_len) + + def incr(self, key, delta=1): + """Increments the value for a key.""" + value = self.get(key) + if value is None: + return None + new_value = int(value) + delta + self.cache[key] = (self.cache[key][0], str(new_value)) + return new_value + + def delete(self, key, time=0): + """Deletes the value associated with a key.""" + if key in self.cache: + del self.cache[key] diff --git a/terracotta/openstack/common/middleware/__init__.py b/terracotta/openstack/common/middleware/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/terracotta/openstack/common/middleware/request_id.py b/terracotta/openstack/common/middleware/request_id.py new file mode 100644 index 0000000..a09e044 --- /dev/null +++ b/terracotta/openstack/common/middleware/request_id.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Compatibility shim for Kilo, while operators migrate to oslo.middleware.""" + +from oslo_middleware import request_id + +from nova.openstack.common import versionutils + + +ENV_REQUEST_ID = 'openstack.request_id' +HTTP_RESP_HEADER_REQUEST_ID = 'x-openstack-request-id' + + +@versionutils.deprecated(as_of=versionutils.deprecated.KILO, + in_favor_of='oslo.middleware.RequestId') +class RequestIdMiddleware(request_id.RequestId): + pass diff --git a/terracotta/openstack/common/periodic_task.py b/terracotta/openstack/common/periodic_task.py new file mode 100644 index 0000000..419098b --- /dev/null +++ b/terracotta/openstack/common/periodic_task.py @@ -0,0 +1,232 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import logging +import random +import time + +from oslo_config import cfg +import six + +from nova.openstack.common._i18n import _, _LE, _LI + + +periodic_opts = [ + cfg.BoolOpt('run_external_periodic_tasks', + default=True, + help='Some periodic tasks can be run in a separate process. ' + 'Should we run them here?'), +] + +CONF = cfg.CONF +CONF.register_opts(periodic_opts) + +LOG = logging.getLogger(__name__) + +DEFAULT_INTERVAL = 60.0 + + +def list_opts(): + """Entry point for oslo-config-generator.""" + return [(None, copy.deepcopy(periodic_opts))] + + +class InvalidPeriodicTaskArg(Exception): + message = _("Unexpected argument for periodic task creation: %(arg)s.") + + +def periodic_task(*args, **kwargs): + """Decorator to indicate that a method is a periodic task. + + This decorator can be used in two ways: + + 1. Without arguments '@periodic_task', this will be run on the default + interval of 60 seconds. + + 2. With arguments: + @periodic_task(spacing=N [, run_immediately=[True|False]] + [, name=[None|"string"]) + this will be run on approximately every N seconds. If this number is + negative the periodic task will be disabled. If the run_immediately + argument is provided and has a value of 'True', the first run of the + task will be shortly after task scheduler starts. If + run_immediately is omitted or set to 'False', the first time the + task runs will be approximately N seconds after the task scheduler + starts. If name is not provided, __name__ of function is used. + """ + def decorator(f): + # Test for old style invocation + if 'ticks_between_runs' in kwargs: + raise InvalidPeriodicTaskArg(arg='ticks_between_runs') + + # Control if run at all + f._periodic_task = True + f._periodic_external_ok = kwargs.pop('external_process_ok', False) + if f._periodic_external_ok and not CONF.run_external_periodic_tasks: + f._periodic_enabled = False + else: + f._periodic_enabled = kwargs.pop('enabled', True) + f._periodic_name = kwargs.pop('name', f.__name__) + + # Control frequency + f._periodic_spacing = kwargs.pop('spacing', 0) + f._periodic_immediate = kwargs.pop('run_immediately', False) + if f._periodic_immediate: + f._periodic_last_run = None + else: + f._periodic_last_run = time.time() + return f + + # NOTE(sirp): The `if` is necessary to allow the decorator to be used with + # and without parenthesis. + # + # In the 'with-parenthesis' case (with kwargs present), this function needs + # to return a decorator function since the interpreter will invoke it like: + # + # periodic_task(*args, **kwargs)(f) + # + # In the 'without-parenthesis' case, the original function will be passed + # in as the first argument, like: + # + # periodic_task(f) + if kwargs: + return decorator + else: + return decorator(args[0]) + + +class _PeriodicTasksMeta(type): + def _add_periodic_task(cls, task): + """Add a periodic task to the list of periodic tasks. + + The task should already be decorated by @periodic_task. + + :return: whether task was actually enabled + """ + name = task._periodic_name + + if task._periodic_spacing < 0: + LOG.info(_LI('Skipping periodic task %(task)s because ' + 'its interval is negative'), + {'task': name}) + return False + if not task._periodic_enabled: + LOG.info(_LI('Skipping periodic task %(task)s because ' + 'it is disabled'), + {'task': name}) + return False + + # A periodic spacing of zero indicates that this task should + # be run on the default interval to avoid running too + # frequently. + if task._periodic_spacing == 0: + task._periodic_spacing = DEFAULT_INTERVAL + + cls._periodic_tasks.append((name, task)) + cls._periodic_spacing[name] = task._periodic_spacing + return True + + def __init__(cls, names, bases, dict_): + """Metaclass that allows us to collect decorated periodic tasks.""" + super(_PeriodicTasksMeta, cls).__init__(names, bases, dict_) + + # NOTE(sirp): if the attribute is not present then we must be the base + # class, so, go ahead an initialize it. If the attribute is present, + # then we're a subclass so make a copy of it so we don't step on our + # parent's toes. + try: + cls._periodic_tasks = cls._periodic_tasks[:] + except AttributeError: + cls._periodic_tasks = [] + + try: + cls._periodic_spacing = cls._periodic_spacing.copy() + except AttributeError: + cls._periodic_spacing = {} + + for value in cls.__dict__.values(): + if getattr(value, '_periodic_task', False): + cls._add_periodic_task(value) + + +def _nearest_boundary(last_run, spacing): + """Find nearest boundary which is in the past, which is a multiple of the + spacing with the last run as an offset. + + Eg if last run was 10 and spacing was 7, the new last run could be: 17, 24, + 31, 38... + + 0% to 5% of the spacing value will be added to this value to ensure tasks + do not synchronize. This jitter is rounded to the nearest second, this + means that spacings smaller than 20 seconds will not have jitter. + """ + current_time = time.time() + if last_run is None: + return current_time + delta = current_time - last_run + offset = delta % spacing + # Add up to 5% jitter + jitter = int(spacing * (random.random() / 20)) + return current_time - offset + jitter + + +@six.add_metaclass(_PeriodicTasksMeta) +class PeriodicTasks(object): + def __init__(self): + super(PeriodicTasks, self).__init__() + self._periodic_last_run = {} + for name, task in self._periodic_tasks: + self._periodic_last_run[name] = task._periodic_last_run + + def add_periodic_task(self, task): + """Add a periodic task to the list of periodic tasks. + + The task should already be decorated by @periodic_task. + """ + if self.__class__._add_periodic_task(task): + self._periodic_last_run[task._periodic_name] = ( + task._periodic_last_run) + + def run_periodic_tasks(self, context, raise_on_error=False): + """Tasks to be run at a periodic interval.""" + idle_for = DEFAULT_INTERVAL + for task_name, task in self._periodic_tasks: + full_task_name = '.'.join([self.__class__.__name__, task_name]) + + spacing = self._periodic_spacing[task_name] + last_run = self._periodic_last_run[task_name] + + # Check if due, if not skip + idle_for = min(idle_for, spacing) + if last_run is not None: + delta = last_run + spacing - time.time() + if delta > 0: + idle_for = min(idle_for, delta) + continue + + LOG.debug("Running periodic task %(full_task_name)s", + {"full_task_name": full_task_name}) + self._periodic_last_run[task_name] = _nearest_boundary( + last_run, spacing) + + try: + task(self, context) + except Exception as e: + if raise_on_error: + raise + LOG.exception(_LE("Error during %(full_task_name)s: %(e)s"), + {"full_task_name": full_task_name, "e": e}) + time.sleep(0) + + return idle_for diff --git a/terracotta/openstack/common/policy.py b/terracotta/openstack/common/policy.py new file mode 100644 index 0000000..abde3a5 --- /dev/null +++ b/terracotta/openstack/common/policy.py @@ -0,0 +1,963 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2012 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Common Policy Engine Implementation + +Policies can be expressed in one of two forms: A list of lists, or a +string written in the new policy language. + +In the list-of-lists representation, each check inside the innermost +list is combined as with an "and" conjunction--for that check to pass, +all the specified checks must pass. These innermost lists are then +combined as with an "or" conjunction. As an example, take the following +rule, expressed in the list-of-lists representation:: + + [["role:admin"], ["project_id:%(project_id)s", "role:projectadmin"]] + +This is the original way of expressing policies, but there now exists a +new way: the policy language. + +In the policy language, each check is specified the same way as in the +list-of-lists representation: a simple "a:b" pair that is matched to +the correct class to perform that check:: + + +===========================================================================+ + | TYPE | SYNTAX | + +===========================================================================+ + |User's Role | role:admin | + +---------------------------------------------------------------------------+ + |Rules already defined on policy | rule:admin_required | + +---------------------------------------------------------------------------+ + |Against URL's¹ | http://my-url.org/check | + +---------------------------------------------------------------------------+ + |User attributes² | project_id:%(target.project.id)s | + +---------------------------------------------------------------------------+ + |Strings | :'xpto2035abc' | + | | 'myproject': | + +---------------------------------------------------------------------------+ + | | project_id:xpto2035abc | + |Literals | domain_id:20 | + | | True:%(user.enabled)s | + +===========================================================================+ + +¹URL checking must return 'True' to be valid +²User attributes (obtained through the token): user_id, domain_id or project_id + +Conjunction operators are available, allowing for more expressiveness +in crafting policies. So, in the policy language, the previous check in +list-of-lists becomes:: + + role:admin or (project_id:%(project_id)s and role:projectadmin) + +The policy language also has the "not" operator, allowing a richer +policy rule:: + + project_id:%(project_id)s and not role:dunce + +Attributes sent along with API calls can be used by the policy engine +(on the right side of the expression), by using the following syntax:: + + :%(user.id)s + +Contextual attributes of objects identified by their IDs are loaded +from the database. They are also available to the policy engine and +can be checked through the `target` keyword:: + + :%(target.role.name)s + +Finally, two special policy checks should be mentioned; the policy +check "@" will always accept an access, and the policy check "!" will +always reject an access. (Note that if a rule is either the empty +list ("[]") or the empty string, this is equivalent to the "@" policy +check.) Of these, the "!" policy check is probably the most useful, +as it allows particular rules to be explicitly disabled. +""" + +import abc +import ast +import copy +import logging +import os +import re + +from oslo_config import cfg +from oslo_serialization import jsonutils +import six +import six.moves.urllib.parse as urlparse +import six.moves.urllib.request as urlrequest + +from nova.openstack.common import fileutils +from nova.openstack.common._i18n import _, _LE + + +policy_opts = [ + cfg.StrOpt('policy_file', + default='policy.json', + help=_('The JSON file that defines policies.')), + cfg.StrOpt('policy_default_rule', + default='default', + help=_('Default rule. Enforced when a requested rule is not ' + 'found.')), + cfg.MultiStrOpt('policy_dirs', + default=['policy.d'], + help=_('Directories where policy configuration files are ' + 'stored. They can be relative to any directory ' + 'in the search path defined by the config_dir ' + 'option, or absolute paths. The file defined by ' + 'policy_file must exist for these directories to ' + 'be searched. Missing or empty directories are ' + 'ignored.')), +] + +CONF = cfg.CONF +CONF.register_opts(policy_opts) + +LOG = logging.getLogger(__name__) + +_checks = {} + + +def list_opts(): + """Entry point for oslo-config-generator.""" + return [(None, copy.deepcopy(policy_opts))] + + +class PolicyNotAuthorized(Exception): + + def __init__(self, rule): + msg = _("Policy doesn't allow %s to be performed.") % rule + super(PolicyNotAuthorized, self).__init__(msg) + + +class Rules(dict): + """A store for rules. Handles the default_rule setting directly.""" + + @classmethod + def load_json(cls, data, default_rule=None): + """Allow loading of JSON rule data.""" + + # Suck in the JSON data and parse the rules + rules = dict((k, parse_rule(v)) for k, v in + jsonutils.loads(data).items()) + + return cls(rules, default_rule) + + def __init__(self, rules=None, default_rule=None): + """Initialize the Rules store.""" + + super(Rules, self).__init__(rules or {}) + self.default_rule = default_rule + + def __missing__(self, key): + """Implements the default rule handling.""" + + if isinstance(self.default_rule, dict): + raise KeyError(key) + + # If the default rule isn't actually defined, do something + # reasonably intelligent + if not self.default_rule: + raise KeyError(key) + + if isinstance(self.default_rule, BaseCheck): + return self.default_rule + + # We need to check this or we can get infinite recursion + if self.default_rule not in self: + raise KeyError(key) + + elif isinstance(self.default_rule, six.string_types): + return self[self.default_rule] + + def __str__(self): + """Dumps a string representation of the rules.""" + + # Start by building the canonical strings for the rules + out_rules = {} + for key, value in self.items(): + # Use empty string for singleton TrueCheck instances + if isinstance(value, TrueCheck): + out_rules[key] = '' + else: + out_rules[key] = str(value) + + # Dump a pretty-printed JSON representation + return jsonutils.dumps(out_rules, indent=4) + + +class Enforcer(object): + """Responsible for loading and enforcing rules. + + :param policy_file: Custom policy file to use, if none is + specified, `CONF.policy_file` will be + used. + :param rules: Default dictionary / Rules to use. It will be + considered just in the first instantiation. If + `load_rules(True)`, `clear()` or `set_rules(True)` + is called this will be overwritten. + :param default_rule: Default rule to use, CONF.default_rule will + be used if none is specified. + :param use_conf: Whether to load rules from cache or config file. + :param overwrite: Whether to overwrite existing rules when reload rules + from config file. + """ + + def __init__(self, policy_file=None, rules=None, + default_rule=None, use_conf=True, overwrite=True): + self.default_rule = default_rule or CONF.policy_default_rule + self.rules = Rules(rules, self.default_rule) + + self.policy_path = None + self.policy_file = policy_file or CONF.policy_file + self.use_conf = use_conf + self.overwrite = overwrite + + def set_rules(self, rules, overwrite=True, use_conf=False): + """Create a new Rules object based on the provided dict of rules. + + :param rules: New rules to use. It should be an instance of dict. + :param overwrite: Whether to overwrite current rules or update them + with the new rules. + :param use_conf: Whether to reload rules from cache or config file. + """ + + if not isinstance(rules, dict): + raise TypeError(_("Rules must be an instance of dict or Rules, " + "got %s instead") % type(rules)) + self.use_conf = use_conf + if overwrite: + self.rules = Rules(rules, self.default_rule) + else: + self.rules.update(rules) + + def clear(self): + """Clears Enforcer rules, policy's cache and policy's path.""" + self.set_rules({}) + fileutils.delete_cached_file(self.policy_path) + self.default_rule = None + self.policy_path = None + + def load_rules(self, force_reload=False): + """Loads policy_path's rules. + + Policy file is cached and will be reloaded if modified. + + :param force_reload: Whether to reload rules from config file. + """ + + if force_reload: + self.use_conf = force_reload + + if self.use_conf: + if not self.policy_path: + self.policy_path = self._get_policy_path(self.policy_file) + + self._load_policy_file(self.policy_path, force_reload, + overwrite=self.overwrite) + for path in CONF.policy_dirs: + try: + path = self._get_policy_path(path) + except cfg.ConfigFilesNotFoundError: + continue + self._walk_through_policy_directory(path, + self._load_policy_file, + force_reload, False) + + @staticmethod + def _walk_through_policy_directory(path, func, *args): + # We do not iterate over sub-directories. + policy_files = next(os.walk(path))[2] + policy_files.sort() + for policy_file in [p for p in policy_files if not p.startswith('.')]: + func(os.path.join(path, policy_file), *args) + + def _load_policy_file(self, path, force_reload, overwrite=True): + reloaded, data = fileutils.read_cached_file( + path, force_reload=force_reload) + if reloaded or not self.rules or not overwrite: + rules = Rules.load_json(data, self.default_rule) + self.set_rules(rules, overwrite=overwrite, use_conf=True) + LOG.debug("Reloaded policy file: %(path)s", + {'path': path}) + + def _get_policy_path(self, path): + """Locate the policy json data file/path. + + :param path: It's value can be a full path or related path. When + full path specified, this function just returns the full + path. When related path specified, this function will + search configuration directories to find one that exists. + + :returns: The policy path + + :raises: ConfigFilesNotFoundError if the file/path couldn't + be located. + """ + policy_path = CONF.find_file(path) + + if policy_path: + return policy_path + + raise cfg.ConfigFilesNotFoundError((path,)) + + def enforce(self, rule, target, creds, do_raise=False, + exc=None, *args, **kwargs): + """Checks authorization of a rule against the target and credentials. + + :param rule: A string or BaseCheck instance specifying the rule + to evaluate. + :param target: As much information about the object being operated + on as possible, as a dictionary. + :param creds: As much information about the user performing the + action as possible, as a dictionary. + :param do_raise: Whether to raise an exception or not if check + fails. + :param exc: Class of the exception to raise if the check fails. + Any remaining arguments passed to enforce() (both + positional and keyword arguments) will be passed to + the exception class. If not specified, PolicyNotAuthorized + will be used. + + :return: Returns False if the policy does not allow the action and + exc is not provided; otherwise, returns a value that + evaluates to True. Note: for rules using the "case" + expression, this True value will be the specified string + from the expression. + """ + + self.load_rules() + + # Allow the rule to be a Check tree + if isinstance(rule, BaseCheck): + result = rule(target, creds, self) + elif not self.rules: + # No rules to reference means we're going to fail closed + result = False + else: + try: + # Evaluate the rule + result = self.rules[rule](target, creds, self) + except KeyError: + LOG.debug("Rule [%s] doesn't exist" % rule) + # If the rule doesn't exist, fail closed + result = False + + # If it is False, raise the exception if requested + if do_raise and not result: + if exc: + raise exc(*args, **kwargs) + + raise PolicyNotAuthorized(rule) + + return result + + +@six.add_metaclass(abc.ABCMeta) +class BaseCheck(object): + """Abstract base class for Check classes.""" + + @abc.abstractmethod + def __str__(self): + """String representation of the Check tree rooted at this node.""" + + pass + + @abc.abstractmethod + def __call__(self, target, cred, enforcer): + """Triggers if instance of the class is called. + + Performs the check. Returns False to reject the access or a + true value (not necessary True) to accept the access. + """ + + pass + + +class FalseCheck(BaseCheck): + """A policy check that always returns False (disallow).""" + + def __str__(self): + """Return a string representation of this check.""" + + return "!" + + def __call__(self, target, cred, enforcer): + """Check the policy.""" + + return False + + +class TrueCheck(BaseCheck): + """A policy check that always returns True (allow).""" + + def __str__(self): + """Return a string representation of this check.""" + + return "@" + + def __call__(self, target, cred, enforcer): + """Check the policy.""" + + return True + + +class Check(BaseCheck): + """A base class to allow for user-defined policy checks.""" + + def __init__(self, kind, match): + """Initiates Check instance. + + :param kind: The kind of the check, i.e., the field before the + ':'. + :param match: The match of the check, i.e., the field after + the ':'. + """ + + self.kind = kind + self.match = match + + def __str__(self): + """Return a string representation of this check.""" + + return "%s:%s" % (self.kind, self.match) + + +class NotCheck(BaseCheck): + """Implements the "not" logical operator. + + A policy check that inverts the result of another policy check. + """ + + def __init__(self, rule): + """Initialize the 'not' check. + + :param rule: The rule to negate. Must be a Check. + """ + + self.rule = rule + + def __str__(self): + """Return a string representation of this check.""" + + return "not %s" % self.rule + + def __call__(self, target, cred, enforcer): + """Check the policy. + + Returns the logical inverse of the wrapped check. + """ + + return not self.rule(target, cred, enforcer) + + +class AndCheck(BaseCheck): + """Implements the "and" logical operator. + + A policy check that requires that a list of other checks all return True. + """ + + def __init__(self, rules): + """Initialize the 'and' check. + + :param rules: A list of rules that will be tested. + """ + + self.rules = rules + + def __str__(self): + """Return a string representation of this check.""" + + return "(%s)" % ' and '.join(str(r) for r in self.rules) + + def __call__(self, target, cred, enforcer): + """Check the policy. + + Requires that all rules accept in order to return True. + """ + + for rule in self.rules: + if not rule(target, cred, enforcer): + return False + + return True + + def add_check(self, rule): + """Adds rule to be tested. + + Allows addition of another rule to the list of rules that will + be tested. Returns the AndCheck object for convenience. + """ + + self.rules.append(rule) + return self + + +class OrCheck(BaseCheck): + """Implements the "or" operator. + + A policy check that requires that at least one of a list of other + checks returns True. + """ + + def __init__(self, rules): + """Initialize the 'or' check. + + :param rules: A list of rules that will be tested. + """ + + self.rules = rules + + def __str__(self): + """Return a string representation of this check.""" + + return "(%s)" % ' or '.join(str(r) for r in self.rules) + + def __call__(self, target, cred, enforcer): + """Check the policy. + + Requires that at least one rule accept in order to return True. + """ + + for rule in self.rules: + if rule(target, cred, enforcer): + return True + return False + + def add_check(self, rule): + """Adds rule to be tested. + + Allows addition of another rule to the list of rules that will + be tested. Returns the OrCheck object for convenience. + """ + + self.rules.append(rule) + return self + + +def _parse_check(rule): + """Parse a single base check rule into an appropriate Check object.""" + + # Handle the special checks + if rule == '!': + return FalseCheck() + elif rule == '@': + return TrueCheck() + + try: + kind, match = rule.split(':', 1) + except Exception: + LOG.exception(_LE("Failed to understand rule %s") % rule) + # If the rule is invalid, we'll fail closed + return FalseCheck() + + # Find what implements the check + if kind in _checks: + return _checks[kind](kind, match) + elif None in _checks: + return _checks[None](kind, match) + else: + LOG.error(_LE("No handler for matches of kind %s") % kind) + return FalseCheck() + + +def _parse_list_rule(rule): + """Translates the old list-of-lists syntax into a tree of Check objects. + + Provided for backwards compatibility. + """ + + # Empty rule defaults to True + if not rule: + return TrueCheck() + + # Outer list is joined by "or"; inner list by "and" + or_list = [] + for inner_rule in rule: + # Elide empty inner lists + if not inner_rule: + continue + + # Handle bare strings + if isinstance(inner_rule, six.string_types): + inner_rule = [inner_rule] + + # Parse the inner rules into Check objects + and_list = [_parse_check(r) for r in inner_rule] + + # Append the appropriate check to the or_list + if len(and_list) == 1: + or_list.append(and_list[0]) + else: + or_list.append(AndCheck(and_list)) + + # If we have only one check, omit the "or" + if not or_list: + return FalseCheck() + elif len(or_list) == 1: + return or_list[0] + + return OrCheck(or_list) + + +# Used for tokenizing the policy language +_tokenize_re = re.compile(r'\s+') + + +def _parse_tokenize(rule): + """Tokenizer for the policy language. + + Most of the single-character tokens are specified in the + _tokenize_re; however, parentheses need to be handled specially, + because they can appear inside a check string. Thankfully, those + parentheses that appear inside a check string can never occur at + the very beginning or end ("%(variable)s" is the correct syntax). + """ + + for tok in _tokenize_re.split(rule): + # Skip empty tokens + if not tok or tok.isspace(): + continue + + # Handle leading parens on the token + clean = tok.lstrip('(') + for i in range(len(tok) - len(clean)): + yield '(', '(' + + # If it was only parentheses, continue + if not clean: + continue + else: + tok = clean + + # Handle trailing parens on the token + clean = tok.rstrip(')') + trail = len(tok) - len(clean) + + # Yield the cleaned token + lowered = clean.lower() + if lowered in ('and', 'or', 'not'): + # Special tokens + yield lowered, clean + elif clean: + # Not a special token, but not composed solely of ')' + if len(tok) >= 2 and ((tok[0], tok[-1]) in + [('"', '"'), ("'", "'")]): + # It's a quoted string + yield 'string', tok[1:-1] + else: + yield 'check', _parse_check(clean) + + # Yield the trailing parens + for i in range(trail): + yield ')', ')' + + +class ParseStateMeta(type): + """Metaclass for the ParseState class. + + Facilitates identifying reduction methods. + """ + + def __new__(mcs, name, bases, cls_dict): + """Create the class. + + Injects the 'reducers' list, a list of tuples matching token sequences + to the names of the corresponding reduction methods. + """ + + reducers = [] + + for key, value in cls_dict.items(): + if not hasattr(value, 'reducers'): + continue + for reduction in value.reducers: + reducers.append((reduction, key)) + + cls_dict['reducers'] = reducers + + return super(ParseStateMeta, mcs).__new__(mcs, name, bases, cls_dict) + + +def reducer(*tokens): + """Decorator for reduction methods. + + Arguments are a sequence of tokens, in order, which should trigger running + this reduction method. + """ + + def decorator(func): + # Make sure we have a list of reducer sequences + if not hasattr(func, 'reducers'): + func.reducers = [] + + # Add the tokens to the list of reducer sequences + func.reducers.append(list(tokens)) + + return func + + return decorator + + +@six.add_metaclass(ParseStateMeta) +class ParseState(object): + """Implement the core of parsing the policy language. + + Uses a greedy reduction algorithm to reduce a sequence of tokens into + a single terminal, the value of which will be the root of the Check tree. + + Note: error reporting is rather lacking. The best we can get with + this parser formulation is an overall "parse failed" error. + Fortunately, the policy language is simple enough that this + shouldn't be that big a problem. + """ + + def __init__(self): + """Initialize the ParseState.""" + + self.tokens = [] + self.values = [] + + def reduce(self): + """Perform a greedy reduction of the token stream. + + If a reducer method matches, it will be executed, then the + reduce() method will be called recursively to search for any more + possible reductions. + """ + + for reduction, methname in self.reducers: + if (len(self.tokens) >= len(reduction) and + self.tokens[-len(reduction):] == reduction): + # Get the reduction method + meth = getattr(self, methname) + + # Reduce the token stream + results = meth(*self.values[-len(reduction):]) + + # Update the tokens and values + self.tokens[-len(reduction):] = [r[0] for r in results] + self.values[-len(reduction):] = [r[1] for r in results] + + # Check for any more reductions + return self.reduce() + + def shift(self, tok, value): + """Adds one more token to the state. Calls reduce().""" + + self.tokens.append(tok) + self.values.append(value) + + # Do a greedy reduce... + self.reduce() + + @property + def result(self): + """Obtain the final result of the parse. + + Raises ValueError if the parse failed to reduce to a single result. + """ + + if len(self.values) != 1: + raise ValueError("Could not parse rule") + return self.values[0] + + @reducer('(', 'check', ')') + @reducer('(', 'and_expr', ')') + @reducer('(', 'or_expr', ')') + def _wrap_check(self, _p1, check, _p2): + """Turn parenthesized expressions into a 'check' token.""" + + return [('check', check)] + + @reducer('check', 'and', 'check') + def _make_and_expr(self, check1, _and, check2): + """Create an 'and_expr'. + + Join two checks by the 'and' operator. + """ + + return [('and_expr', AndCheck([check1, check2]))] + + @reducer('and_expr', 'and', 'check') + def _extend_and_expr(self, and_expr, _and, check): + """Extend an 'and_expr' by adding one more check.""" + + return [('and_expr', and_expr.add_check(check))] + + @reducer('check', 'or', 'check') + def _make_or_expr(self, check1, _or, check2): + """Create an 'or_expr'. + + Join two checks by the 'or' operator. + """ + + return [('or_expr', OrCheck([check1, check2]))] + + @reducer('or_expr', 'or', 'check') + def _extend_or_expr(self, or_expr, _or, check): + """Extend an 'or_expr' by adding one more check.""" + + return [('or_expr', or_expr.add_check(check))] + + @reducer('not', 'check') + def _make_not_expr(self, _not, check): + """Invert the result of another check.""" + + return [('check', NotCheck(check))] + + +def _parse_text_rule(rule): + """Parses policy to the tree. + + Translates a policy written in the policy language into a tree of + Check objects. + """ + + # Empty rule means always accept + if not rule: + return TrueCheck() + + # Parse the token stream + state = ParseState() + for tok, value in _parse_tokenize(rule): + state.shift(tok, value) + + try: + return state.result + except ValueError: + # Couldn't parse the rule + LOG.exception(_LE("Failed to understand rule %s") % rule) + + # Fail closed + return FalseCheck() + + +def parse_rule(rule): + """Parses a policy rule into a tree of Check objects.""" + + # If the rule is a string, it's in the policy language + if isinstance(rule, six.string_types): + return _parse_text_rule(rule) + return _parse_list_rule(rule) + + +def register(name, func=None): + """Register a function or Check class as a policy check. + + :param name: Gives the name of the check type, e.g., 'rule', + 'role', etc. If name is None, a default check type + will be registered. + :param func: If given, provides the function or class to register. + If not given, returns a function taking one argument + to specify the function or class to register, + allowing use as a decorator. + """ + + # Perform the actual decoration by registering the function or + # class. Returns the function or class for compliance with the + # decorator interface. + def decorator(func): + _checks[name] = func + return func + + # If the function or class is given, do the registration + if func: + return decorator(func) + + return decorator + + +@register("rule") +class RuleCheck(Check): + def __call__(self, target, creds, enforcer): + """Recursively checks credentials based on the defined rules.""" + + try: + return enforcer.rules[self.match](target, creds, enforcer) + except KeyError: + # We don't have any matching rule; fail closed + return False + + +@register("role") +class RoleCheck(Check): + def __call__(self, target, creds, enforcer): + """Check that there is a matching role in the cred dict.""" + + return self.match.lower() in [x.lower() for x in creds['roles']] + + +@register('http') +class HttpCheck(Check): + def __call__(self, target, creds, enforcer): + """Check http: rules by calling to a remote server. + + This example implementation simply verifies that the response + is exactly 'True'. + """ + + url = ('http:' + self.match) % target + + # Convert instances of object() in target temporarily to + # empty dict to avoid circular reference detection + # errors in jsonutils.dumps(). + temp_target = copy.deepcopy(target) + for key in target.keys(): + element = target.get(key) + if type(element) is object: + temp_target[key] = {} + + data = {'target': jsonutils.dumps(temp_target), + 'credentials': jsonutils.dumps(creds)} + post_data = urlparse.urlencode(data) + f = urlrequest.urlopen(url, post_data) + return f.read() == "True" + + +@register(None) +class GenericCheck(Check): + def __call__(self, target, creds, enforcer): + """Check an individual match. + + Matches look like: + + tenant:%(tenant_id)s + role:compute:admin + True:%(user.enabled)s + 'Member':%(role.name)s + """ + + try: + match = self.match % target + except KeyError: + # While doing GenericCheck if key not + # present in Target return false + return False + + try: + # Try to interpret self.kind as a literal + leftval = ast.literal_eval(self.kind) + except ValueError: + try: + kind_parts = self.kind.split('.') + leftval = creds + for kind_part in kind_parts: + leftval = leftval[kind_part] + except KeyError: + return False + return match == six.text_type(leftval) diff --git a/terracotta/openstack/common/report/__init__.py b/terracotta/openstack/common/report/__init__.py new file mode 100644 index 0000000..35390ec --- /dev/null +++ b/terracotta/openstack/common/report/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides a way to generate serializable reports + +This package/module provides mechanisms for defining reports +which may then be serialized into various data types. Each +report ( :class:`openstack.common.report.report.BasicReport` ) +is composed of one or more report sections +( :class:`openstack.common.report.report.BasicSection` ), +which contain generators which generate data models +( :class:`openstack.common.report.models.base.ReportModels` ), +which are then serialized by views. +""" diff --git a/terracotta/openstack/common/report/generators/__init__.py b/terracotta/openstack/common/report/generators/__init__.py new file mode 100644 index 0000000..68473f2 --- /dev/null +++ b/terracotta/openstack/common/report/generators/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides Data Model Generators + +This module defines classes for generating data models +( :class:`openstack.common.report.models.base.ReportModel` ). +A generator is any object which is callable with no parameters +and returns a data model. +""" diff --git a/terracotta/openstack/common/report/generators/conf.py b/terracotta/openstack/common/report/generators/conf.py new file mode 100644 index 0000000..4d73514 --- /dev/null +++ b/terracotta/openstack/common/report/generators/conf.py @@ -0,0 +1,44 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides OpenStack config generators + +This module defines a class for configuration +generators for generating the model in +:mod:`openstack.common.report.models.conf`. +""" + +from oslo_config import cfg + +from nova.openstack.common.report.models import conf as cm + + +class ConfigReportGenerator(object): + """A Configuration Data Generator + + This generator returns + :class:`openstack.common.report.models.conf.ConfigModel`, + by default using the configuration options stored + in :attr:`oslo_config.cfg.CONF`, which is where + OpenStack stores everything. + + :param cnf: the configuration option object + :type cnf: :class:`oslo_config.cfg.ConfigOpts` + """ + + def __init__(self, cnf=cfg.CONF): + self.conf_obj = cnf + + def __call__(self): + return cm.ConfigModel(self.conf_obj) diff --git a/terracotta/openstack/common/report/generators/process.py b/terracotta/openstack/common/report/generators/process.py new file mode 100644 index 0000000..b7cee4c --- /dev/null +++ b/terracotta/openstack/common/report/generators/process.py @@ -0,0 +1,38 @@ +# Copyright 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides process-data generators + +This modules defines a class for generating +process data by way of the psutil package. +""" + +import os + +import psutil + +from nova.openstack.common.report.models import process as pm + + +class ProcessReportGenerator(object): + """A Process Data Generator + + This generator returns a + :class:`openstack.common.report.models.process.ProcessModel` + based on the current process (which will also include + all subprocesses, recursively) using the :class:`psutil.Process` class`. + """ + + def __call__(self): + return pm.ProcessModel(psutil.Process(os.getpid())) diff --git a/terracotta/openstack/common/report/generators/threading.py b/terracotta/openstack/common/report/generators/threading.py new file mode 100644 index 0000000..a5b6fe0 --- /dev/null +++ b/terracotta/openstack/common/report/generators/threading.py @@ -0,0 +1,86 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides thread-related generators + +This module defines classes for threading-related +generators for generating the models in +:mod:`openstack.common.report.models.threading`. +""" + +from __future__ import absolute_import + +import sys +import threading + +import greenlet + +from nova.openstack.common.report.models import threading as tm +from nova.openstack.common.report.models import with_default_views as mwdv +from nova.openstack.common.report import utils as rutils +from nova.openstack.common.report.views.text import generic as text_views + + +class ThreadReportGenerator(object): + """A Thread Data Generator + + This generator returns a collection of + :class:`openstack.common.report.models.threading.ThreadModel` + objects by introspecting the current python state using + :func:`sys._current_frames()` . Its constructor may optionally + be passed a frame object. This frame object will be interpreted + as the actual stack trace for the current thread, and, come generation + time, will be used to replace the stack trace of the thread in which + this code is running. + """ + + def __init__(self, curr_thread_traceback=None): + self.traceback = curr_thread_traceback + + def __call__(self): + threadModels = dict( + (thread_id, tm.ThreadModel(thread_id, stack)) + for thread_id, stack in sys._current_frames().items() + ) + + if self.traceback is not None: + curr_thread_id = threading.current_thread().ident + threadModels[curr_thread_id] = tm.ThreadModel(curr_thread_id, + self.traceback) + + return mwdv.ModelWithDefaultViews(threadModels, + text_view=text_views.MultiView()) + + +class GreenThreadReportGenerator(object): + """A Green Thread Data Generator + + This generator returns a collection of + :class:`openstack.common.report.models.threading.GreenThreadModel` + objects by introspecting the current python garbage collection + state, and sifting through for :class:`greenlet.greenlet` objects. + + .. seealso:: + + Function :func:`openstack.common.report.utils._find_objects` + """ + + def __call__(self): + threadModels = [ + tm.GreenThreadModel(gr.gr_frame) + for gr in rutils._find_objects(greenlet.greenlet) + ] + + return mwdv.ModelWithDefaultViews(threadModels, + text_view=text_views.MultiView()) diff --git a/terracotta/openstack/common/report/generators/version.py b/terracotta/openstack/common/report/generators/version.py new file mode 100644 index 0000000..65302d5 --- /dev/null +++ b/terracotta/openstack/common/report/generators/version.py @@ -0,0 +1,46 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides OpenStack version generators + +This module defines a class for OpenStack +version and package information +generators for generating the model in +:mod:`openstack.common.report.models.version`. +""" + +from nova.openstack.common.report.models import version as vm + + +class PackageReportGenerator(object): + """A Package Information Data Generator + + This generator returns + :class:`openstack.common.report.models.version.PackageModel`, + extracting data from the given version object, which should follow + the general format defined in Nova's version information (i.e. it + should contain the methods vendor_string, product_string, and + version_string_with_package). + + :param version_object: the version information object + """ + + def __init__(self, version_obj): + self.version_obj = version_obj + + def __call__(self): + return vm.PackageModel( + self.version_obj.vendor_string(), + self.version_obj.product_string(), + self.version_obj.version_string_with_package()) diff --git a/terracotta/openstack/common/report/guru_meditation_report.py b/terracotta/openstack/common/report/guru_meditation_report.py new file mode 100644 index 0000000..6d1f2e7 --- /dev/null +++ b/terracotta/openstack/common/report/guru_meditation_report.py @@ -0,0 +1,226 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides Guru Meditation Report + +This module defines the actual OpenStack Guru Meditation +Report class. + +This can be used in the OpenStack command definition files. +For example, in a nova command module (under nova/cmd): + +.. code-block:: python + :emphasize-lines: 8,9,10 + + CONF = cfg.CONF + # maybe import some options here... + + def main(): + config.parse_args(sys.argv) + logging.setup('blah') + + TextGuruMeditation.register_section('Some Special Section', + special_section_generator) + TextGuruMeditation.setup_autorun(version_object) + + server = service.Service.create(binary='some-service', + topic=CONF.some_service_topic) + service.serve(server) + service.wait() + +Then, you can do + +.. code-block:: bash + + $ kill -USR1 $SERVICE_PID + +and get a Guru Meditation Report in the file or terminal +where stderr is logged for that given service. +""" + +from __future__ import print_function + +import inspect +import os +import signal +import sys + +from oslo_utils import timeutils + +from nova.openstack.common.report.generators import conf as cgen +from nova.openstack.common.report.generators import process as prgen +from nova.openstack.common.report.generators import threading as tgen +from nova.openstack.common.report.generators import version as pgen +from nova.openstack.common.report import report + + +class GuruMeditation(object): + """A Guru Meditation Report Mixin/Base Class + + This class is a base class for Guru Meditation Reports. + It provides facilities for registering sections and + setting up functionality to auto-run the report on + a certain signal. + + This class should always be used in conjunction with + a Report class via multiple inheritance. It should + always come first in the class list to ensure the + MRO is correct. + """ + + timestamp_fmt = "%Y%m%d%H%M%S" + + def __init__(self, version_obj, sig_handler_tb=None, *args, **kwargs): + self.version_obj = version_obj + self.traceback = sig_handler_tb + + super(GuruMeditation, self).__init__(*args, **kwargs) + self.start_section_index = len(self.sections) + + @classmethod + def register_section(cls, section_title, generator): + """Register a New Section + + This method registers a persistent section for the current + class. + + :param str section_title: the title of the section + :param generator: the generator for the section + """ + + try: + cls.persistent_sections.append([section_title, generator]) + except AttributeError: + cls.persistent_sections = [[section_title, generator]] + + @classmethod + def setup_autorun(cls, version, service_name=None, + log_dir=None, signum=None): + """Set Up Auto-Run + + This method sets up the Guru Meditation Report to automatically + get dumped to stderr or a file in a given dir when the given signal + is received. + + :param version: the version object for the current product + :param service_name: this program name used to construct logfile name + :param logdir: path to a log directory where to create a file + :param signum: the signal to associate with running the report + """ + + if not signum and hasattr(signal, 'SIGUSR1'): + # SIGUSR1 is not supported on all platforms + signum = signal.SIGUSR1 + + if signum: + signal.signal(signum, + lambda sn, tb: cls.handle_signal( + version, service_name, log_dir, tb)) + + @classmethod + def handle_signal(cls, version, service_name, log_dir, traceback): + """The Signal Handler + + This method (indirectly) handles receiving a registered signal and + dumping the Guru Meditation Report to stderr or a file in a given dir. + If service name and log dir are not None, the report will be dumped to + a file named $service_name_gurumeditation_$current_time in the log_dir + directory. + This method is designed to be curried into a proper signal handler by + currying out the version + parameter. + + :param version: the version object for the current product + :param service_name: this program name used to construct logfile name + :param logdir: path to a log directory where to create a file + :param traceback: the traceback provided to the signal handler + """ + + try: + res = cls(version, traceback).run() + except Exception: + print("Unable to run Guru Meditation Report!", + file=sys.stderr) + else: + if log_dir: + service_name = service_name or os.path.basename( + inspect.stack()[-1][1]) + filename = "%s_gurumeditation_%s" % ( + service_name, timeutils.strtime(fmt=cls.timestamp_fmt)) + filepath = os.path.join(log_dir, filename) + try: + with open(filepath, "w") as dumpfile: + dumpfile.write(res) + except Exception: + print("Unable to dump Guru Meditation Report to file %s" % + (filepath,), file=sys.stderr) + else: + print(res, file=sys.stderr) + + def _readd_sections(self): + del self.sections[self.start_section_index:] + + self.add_section('Package', + pgen.PackageReportGenerator(self.version_obj)) + + self.add_section('Threads', + tgen.ThreadReportGenerator(self.traceback)) + + self.add_section('Green Threads', + tgen.GreenThreadReportGenerator()) + + self.add_section('Processes', + prgen.ProcessReportGenerator()) + + self.add_section('Configuration', + cgen.ConfigReportGenerator()) + + try: + for section_title, generator in self.persistent_sections: + self.add_section(section_title, generator) + except AttributeError: + pass + + def run(self): + self._readd_sections() + return super(GuruMeditation, self).run() + + +# GuruMeditation must come first to get the correct MRO +class TextGuruMeditation(GuruMeditation, report.TextReport): + """A Text Guru Meditation Report + + This report is the basic human-readable Guru Meditation Report + + It contains the following sections by default + (in addition to any registered persistent sections): + + - Package Information + + - Threads List + + - Green Threads List + + - Process List + + - Configuration Options + + :param version_obj: the version object for the current product + :param traceback: an (optional) frame object providing the actual + traceback for the current thread + """ + + def __init__(self, version_obj, traceback=None): + super(TextGuruMeditation, self).__init__(version_obj, traceback, + 'Guru Meditation') diff --git a/terracotta/openstack/common/report/models/__init__.py b/terracotta/openstack/common/report/models/__init__.py new file mode 100644 index 0000000..7bfed3d --- /dev/null +++ b/terracotta/openstack/common/report/models/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides data models + +This module provides both the base data model, +as well as several predefined specific data models +to be used in reports. +""" diff --git a/terracotta/openstack/common/report/models/base.py b/terracotta/openstack/common/report/models/base.py new file mode 100644 index 0000000..d840c5b --- /dev/null +++ b/terracotta/openstack/common/report/models/base.py @@ -0,0 +1,162 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides the base report model + +This module defines a class representing the basic report +data model from which all data models should inherit (or +at least implement similar functionality). Data models +store unserialized data generated by generators during +the report serialization process. +""" + +import collections as col +import copy + +import six + + +class ReportModel(col.MutableMapping): + """A Report Data Model + + A report data model contains data generated by some + generator method or class. Data may be read or written + using dictionary-style access, and may be read (but not + written) using object-member-style access. Additionally, + a data model may have an associated view. This view is + used to serialize the model when str() is called on the + model. An appropriate object for a view is callable with + a single parameter: the model to be serialized. + + If present, the object passed in as data will be transformed + into a standard python dict. For mappings, this is fairly + straightforward. For sequences, the indices become keys + and the items become values. + + :param data: a sequence or mapping of data to associate with the model + :param attached_view: a view object to attach to this model + """ + + def __init__(self, data=None, attached_view=None): + self.attached_view = attached_view + + if data is not None: + if isinstance(data, col.Mapping): + self.data = dict(data) + elif isinstance(data, col.Sequence): + # convert a list [a, b, c] to a dict {0: a, 1: b, 2: c} + self.data = dict(enumerate(data)) + else: + raise TypeError('Data for the model must be a sequence ' + 'or mapping.') + else: + self.data = {} + + def __str__(self): + self_cpy = copy.deepcopy(self) + for key in self_cpy: + if getattr(self_cpy[key], 'attached_view', None) is not None: + self_cpy[key] = str(self_cpy[key]) + + if self.attached_view is not None: + return self.attached_view(self_cpy) + else: + raise Exception("Cannot stringify model: no attached view") + + def __repr__(self): + if self.attached_view is not None: + return ("").format(cl=type(self), + dt=self.data, + vw=type(self.attached_view)) + else: + return ("").format(cl=type(self), + dt=self.data) + + def __getitem__(self, attrname): + return self.data[attrname] + + def __setitem__(self, attrname, attrval): + self.data[attrname] = attrval + + def __delitem__(self, attrname): + del self.data[attrname] + + def __contains__(self, key): + return self.data.__contains__(key) + + def __getattr__(self, attrname): + # Needed for deepcopy in Python3. That will avoid an infinite loop + # in __getattr__ . + if 'data' not in self.__dict__: + self.data = {} + + try: + return self.data[attrname] + except KeyError: + # we don't have that key in data, and the + # model class doesn't have that attribute + raise AttributeError( + "'{cl}' object has no attribute '{an}'".format( + cl=type(self).__name__, an=attrname + ) + ) + + def __len__(self): + return len(self.data) + + def __iter__(self): + return self.data.__iter__() + + def set_current_view_type(self, tp, visited=None): + """Set the current view type + + This method attempts to set the current view + type for this model and all submodels by calling + itself recursively on all values, traversing + intervening sequences and mappings when possible, + and ignoring all other objects. + + :param tp: the type of the view ('text', 'json', 'xml', etc) + :param visited: a set of object ids for which the corresponding objects + have already had their view type set + """ + + if visited is None: + visited = set() + + def traverse_obj(obj): + oid = id(obj) + + # don't die on recursive structures, + # and don't treat strings like sequences + if oid in visited or isinstance(obj, six.string_types): + return + + visited.add(oid) + + if hasattr(obj, 'set_current_view_type'): + obj.set_current_view_type(tp, visited=visited) + + if isinstance(obj, col.Sequence): + for item in obj: + traverse_obj(item) + + elif isinstance(obj, col.Mapping): + for val in six.itervalues(obj): + traverse_obj(val) + + traverse_obj(self) diff --git a/terracotta/openstack/common/report/models/conf.py b/terracotta/openstack/common/report/models/conf.py new file mode 100644 index 0000000..ff99ca7 --- /dev/null +++ b/terracotta/openstack/common/report/models/conf.py @@ -0,0 +1,66 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides OpenStack Configuration Model + +This module defines a class representing the data +model for :mod:`oslo_config` configuration options +""" + +from nova.openstack.common.report.models import with_default_views as mwdv +from nova.openstack.common.report.views.text import generic as generic_text_views + + +class ConfigModel(mwdv.ModelWithDefaultViews): + """A Configuration Options Model + + This model holds data about a set of configuration options + from :mod:`oslo_config`. It supports both the default group + of options and named option groups. + + :param conf_obj: a configuration object + :type conf_obj: :class:`oslo_config.cfg.ConfigOpts` + """ + + def __init__(self, conf_obj): + kv_view = generic_text_views.KeyValueView(dict_sep=": ", + before_dict='') + super(ConfigModel, self).__init__(text_view=kv_view) + + def opt_title(optname, co): + return co._opts[optname]['opt'].name + + def opt_value(opt_obj, value): + if opt_obj['opt'].secret: + return '***' + else: + return value + + self['default'] = dict( + (opt_title(optname, conf_obj), + opt_value(conf_obj._opts[optname], conf_obj[optname])) + for optname in conf_obj._opts + ) + + groups = {} + for groupname in conf_obj._groups: + group_obj = conf_obj._groups[groupname] + curr_group_opts = dict( + (opt_title(optname, group_obj), + opt_value(group_obj._opts[optname], + conf_obj[groupname][optname])) + for optname in group_obj._opts) + groups[group_obj.name] = curr_group_opts + + self.update(groups) diff --git a/terracotta/openstack/common/report/models/process.py b/terracotta/openstack/common/report/models/process.py new file mode 100644 index 0000000..b4432e4 --- /dev/null +++ b/terracotta/openstack/common/report/models/process.py @@ -0,0 +1,62 @@ +# Copyright 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides a process model + +This module defines a class representing a process, +potentially with subprocesses. +""" + +import nova.openstack.common.report.models.with_default_views as mwdv +import nova.openstack.common.report.views.text.process as text_views + + +class ProcessModel(mwdv.ModelWithDefaultViews): + """A Process Model + + This model holds data about a process, + including references to any subprocesses + + :param process: a :class:`psutil.Process` object + """ + + def __init__(self, process): + super(ProcessModel, self).__init__( + text_view=text_views.ProcessView()) + + self['pid'] = process.pid + self['parent_pid'] = process.ppid + if hasattr(process, 'uids'): + self['uids'] = {'real': process.uids.real, + 'effective': process.uids.effective, + 'saved': process.uids.saved} + else: + self['uids'] = {'real': None, + 'effective': None, + 'saved': None} + + if hasattr(process, 'gids'): + self['gids'] = {'real': process.gids.real, + 'effective': process.gids.effective, + 'saved': process.gids.saved} + else: + self['gids'] = {'real': None, + 'effective': None, + 'saved': None} + + self['username'] = process.username + self['command'] = process.cmdline + self['state'] = process.status + + self['children'] = [ProcessModel(pr) for pr in process.get_children()] diff --git a/terracotta/openstack/common/report/models/threading.py b/terracotta/openstack/common/report/models/threading.py new file mode 100644 index 0000000..fc386cb --- /dev/null +++ b/terracotta/openstack/common/report/models/threading.py @@ -0,0 +1,100 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides threading and stack-trace models + +This module defines classes representing thread, green +thread, and stack trace data models +""" + +import traceback + +from nova.openstack.common.report.models import with_default_views as mwdv +from nova.openstack.common.report.views.text import threading as text_views + + +class StackTraceModel(mwdv.ModelWithDefaultViews): + """A Stack Trace Model + + This model holds data from a python stack trace, + commonly extracted from running thread information + + :param stack_state: the python stack_state object + """ + + def __init__(self, stack_state): + super(StackTraceModel, self).__init__( + text_view=text_views.StackTraceView()) + + if (stack_state is not None): + self['lines'] = [ + {'filename': fn, 'line': ln, 'name': nm, 'code': cd} + for fn, ln, nm, cd in traceback.extract_stack(stack_state) + ] + # FIXME(flepied): under Python3 f_exc_type doesn't exist + # anymore so we lose information about exceptions + if getattr(stack_state, 'f_exc_type', None) is not None: + self['root_exception'] = { + 'type': stack_state.f_exc_type, + 'value': stack_state.f_exc_value} + else: + self['root_exception'] = None + else: + self['lines'] = [] + self['root_exception'] = None + + +class ThreadModel(mwdv.ModelWithDefaultViews): + """A Thread Model + + This model holds data for information about an + individual thread. It holds both a thread id, + as well as a stack trace for the thread + + .. seealso:: + + Class :class:`StackTraceModel` + + :param int thread_id: the id of the thread + :param stack: the python stack state for the current thread + """ + + # threadId, stack in sys._current_frams().items() + def __init__(self, thread_id, stack): + super(ThreadModel, self).__init__(text_view=text_views.ThreadView()) + + self['thread_id'] = thread_id + self['stack_trace'] = StackTraceModel(stack) + + +class GreenThreadModel(mwdv.ModelWithDefaultViews): + """A Green Thread Model + + This model holds data for information about an + individual thread. Unlike the thread model, + it holds just a stack trace, since green threads + do not have thread ids. + + .. seealso:: + + Class :class:`StackTraceModel` + + :param stack: the python stack state for the green thread + """ + + # gr in greenpool.coroutines_running --> gr.gr_frame + def __init__(self, stack): + super(GreenThreadModel, self).__init__( + {'stack_trace': StackTraceModel(stack)}, + text_view=text_views.GreenThreadView()) diff --git a/terracotta/openstack/common/report/models/version.py b/terracotta/openstack/common/report/models/version.py new file mode 100644 index 0000000..2d8ea1d --- /dev/null +++ b/terracotta/openstack/common/report/models/version.py @@ -0,0 +1,44 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides OpenStack Version Info Model + +This module defines a class representing the data +model for OpenStack package and version information +""" + +from nova.openstack.common.report.models import with_default_views as mwdv +from nova.openstack.common.report.views.text import generic as generic_text_views + + +class PackageModel(mwdv.ModelWithDefaultViews): + """A Package Information Model + + This model holds information about the current + package. It contains vendor, product, and version + information. + + :param str vendor: the product vendor + :param str product: the product name + :param str version: the product version + """ + + def __init__(self, vendor, product, version): + super(PackageModel, self).__init__( + text_view=generic_text_views.KeyValueView() + ) + + self['vendor'] = vendor + self['product'] = product + self['version'] = version diff --git a/terracotta/openstack/common/report/models/with_default_views.py b/terracotta/openstack/common/report/models/with_default_views.py new file mode 100644 index 0000000..0e16572 --- /dev/null +++ b/terracotta/openstack/common/report/models/with_default_views.py @@ -0,0 +1,81 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +from nova.openstack.common.report.models import base as base_model +from nova.openstack.common.report.views.json import generic as jsonviews +from nova.openstack.common.report.views.text import generic as textviews +from nova.openstack.common.report.views.xml import generic as xmlviews + + +class ModelWithDefaultViews(base_model.ReportModel): + """A Model With Default Views of Various Types + + A model with default views has several predefined views, + each associated with a given type. This is often used for + when a submodel should have an attached view, but the view + differs depending on the serialization format + + Parameters are as the superclass, except for any + parameters ending in '_view': these parameters + get stored as default views. + + The default 'default views' are + + text + :class:`openstack.common.report.views.text.generic.KeyValueView` + xml + :class:`openstack.common.report.views.xml.generic.KeyValueView` + json + :class:`openstack.common.report.views.json.generic.KeyValueView` + + .. function:: to_type() + + ('type' is one of the 'default views' defined for this model) + Serializes this model using the default view for 'type' + + :rtype: str + :returns: this model serialized as 'type' + """ + + def __init__(self, *args, **kwargs): + self.views = { + 'text': textviews.KeyValueView(), + 'json': jsonviews.KeyValueView(), + 'xml': xmlviews.KeyValueView() + } + + newargs = copy.copy(kwargs) + for k in kwargs: + if k.endswith('_view'): + self.views[k[:-5]] = kwargs[k] + del newargs[k] + super(ModelWithDefaultViews, self).__init__(*args, **newargs) + + def set_current_view_type(self, tp, visited=None): + self.attached_view = self.views[tp] + super(ModelWithDefaultViews, self).set_current_view_type(tp, visited) + + def __getattr__(self, attrname): + if attrname[:3] == 'to_': + if self.views[attrname[3:]] is not None: + return lambda: self.views[attrname[3:]](self) + else: + raise NotImplementedError(( + "Model {cn.__module__}.{cn.__name__} does not have" + + " a default view for " + "{tp}").format(cn=type(self), tp=attrname[3:])) + else: + return super(ModelWithDefaultViews, self).__getattr__(attrname) diff --git a/terracotta/openstack/common/report/report.py b/terracotta/openstack/common/report/report.py new file mode 100644 index 0000000..9b4ae4b --- /dev/null +++ b/terracotta/openstack/common/report/report.py @@ -0,0 +1,187 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides Report classes + +This module defines various classes representing reports and report sections. +All reports take the form of a report class containing various report +sections. +""" + +from nova.openstack.common.report.views.text import header as header_views + + +class BasicReport(object): + """A Basic Report + + A Basic Report consists of a collection of :class:`ReportSection` + objects, each of which contains a top-level model and generator. + It collects these sections into a cohesive report which may then + be serialized by calling :func:`run`. + """ + + def __init__(self): + self.sections = [] + self._state = 0 + + def add_section(self, view, generator, index=None): + """Add a section to the report + + This method adds a section with the given view and + generator to the report. An index may be specified to + insert the section at a given location in the list; + If no index is specified, the section is appended to the + list. The view is called on the model which results from + the generator when the report is run. A generator is simply + a method or callable object which takes no arguments and + returns a :class:`openstack.common.report.models.base.ReportModel` + or similar object. + + :param view: the top-level view for the section + :param generator: the method or class which generates the model + :param index: the index at which to insert the section + (or None to append it) + :type index: int or None + """ + + if index is None: + self.sections.append(ReportSection(view, generator)) + else: + self.sections.insert(index, ReportSection(view, generator)) + + def run(self): + """Run the report + + This method runs the report, having each section generate + its data and serialize itself before joining the sections + together. The BasicReport accomplishes the joining + by joining the serialized sections together with newlines. + + :rtype: str + :returns: the serialized report + """ + + return "\n".join(str(sect) for sect in self.sections) + + +class ReportSection(object): + """A Report Section + + A report section contains a generator and a top-level view. When something + attempts to serialize the section by calling str() on it, the section runs + the generator and calls the view on the resulting model. + + .. seealso:: + + Class :class:`BasicReport` + :func:`BasicReport.add_section` + + :param view: the top-level view for this section + :param generator: the generator for this section + (any callable object which takes no parameters and returns a data model) + """ + + def __init__(self, view, generator): + self.view = view + self.generator = generator + + def __str__(self): + return self.view(self.generator()) + + +class ReportOfType(BasicReport): + """A Report of a Certain Type + + A ReportOfType has a predefined type associated with it. + This type is automatically propagated down to the each of + the sections upon serialization by wrapping the generator + for each section. + + .. seealso:: + + Class :class:`openstack.common.report.models.with_default_view.ModelWithDefaultView` # noqa + (the entire class) + + Class :class:`openstack.common.report.models.base.ReportModel` + :func:`openstack.common.report.models.base.ReportModel.set_current_view_type` # noqa + + :param str tp: the type of the report + """ + + def __init__(self, tp): + self.output_type = tp + super(ReportOfType, self).__init__() + + def add_section(self, view, generator, index=None): + def with_type(gen): + def newgen(): + res = gen() + try: + res.set_current_view_type(self.output_type) + except AttributeError: + pass + + return res + return newgen + + super(ReportOfType, self).add_section( + view, + with_type(generator), + index + ) + + +class TextReport(ReportOfType): + """A Human-Readable Text Report + + This class defines a report that is designed to be read by a human + being. It has nice section headers, and a formatted title. + + :param str name: the title of the report + """ + + def __init__(self, name): + super(TextReport, self).__init__('text') + self.name = name + # add a title with a generator that creates an empty result model + self.add_section(name, lambda: ('|' * 72) + "\n\n") + + def add_section(self, heading, generator, index=None): + """Add a section to the report + + This method adds a section with the given title, and + generator to the report. An index may be specified to + insert the section at a given location in the list; + If no index is specified, the section is appended to the + list. The view is called on the model which results from + the generator when the report is run. A generator is simply + a method or callable object which takes no arguments and + returns a :class:`openstack.common.report.models.base.ReportModel` + or similar object. + + The model is told to serialize as text (if possible) at serialization + time by wrapping the generator. The view model's attached view + (if any) is wrapped in a + :class:`openstack.common.report.views.text.header.TitledView` + + :param str heading: the title for the section + :param generator: the method or class which generates the model + :param index: the index at which to insert the section + (or None to append) + :type index: int or None + """ + + super(TextReport, self).add_section(header_views.TitledView(heading), + generator, + index) diff --git a/terracotta/openstack/common/report/utils.py b/terracotta/openstack/common/report/utils.py new file mode 100644 index 0000000..fb71e36 --- /dev/null +++ b/terracotta/openstack/common/report/utils.py @@ -0,0 +1,46 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Various utilities for report generation + +This module includes various utilities +used in generating reports. +""" + +import gc + + +class StringWithAttrs(str): + """A String that can have arbitrary attributes + """ + + pass + + +def _find_objects(t): + """Find Objects in the GC State + + This horribly hackish method locates objects of a + given class in the current python instance's garbage + collection state. In case you couldn't tell, this is + horribly hackish, but is necessary for locating all + green threads, since they don't keep track of themselves + like normal threads do in python. + + :param class t: the class of object to locate + :rtype: list + :returns: a list of objects of the given type + """ + + return [o for o in gc.get_objects() if isinstance(o, t)] diff --git a/terracotta/openstack/common/report/views/__init__.py b/terracotta/openstack/common/report/views/__init__.py new file mode 100644 index 0000000..612959b --- /dev/null +++ b/terracotta/openstack/common/report/views/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides predefined views + +This module provides a collection of predefined views +for use in reports. It is separated by type (xml, json, or text). +Each type contains a submodule called 'generic' containing +several basic, universal views for that type. There is also +a predefined view that utilizes Jinja. +""" diff --git a/terracotta/openstack/common/report/views/jinja_view.py b/terracotta/openstack/common/report/views/jinja_view.py new file mode 100644 index 0000000..5f57dc3 --- /dev/null +++ b/terracotta/openstack/common/report/views/jinja_view.py @@ -0,0 +1,137 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides Jinja Views + +This module provides views that utilize the Jinja templating +system for serialization. For more information on Jinja, please +see http://jinja.pocoo.org/ . +""" + +import copy + +import jinja2 + + +class JinjaView(object): + """A Jinja View + + This view renders the given model using the provided Jinja + template. The template can be given in various ways. + If the `VIEw_TEXT` property is defined, that is used as template. + Othewise, if a `path` parameter is passed to the constructor, that + is used to load a file containing the template. If the `path` + parameter is None, the `text` parameter is used as the template. + + The leading newline character and trailing newline character are stripped + from the template (provided they exist). Baseline indentation is + also stripped from each line. The baseline indentation is determined by + checking the indentation of the first line, after stripping off the leading + newline (if any). + + :param str path: the path to the Jinja template + :param str text: the text of the Jinja template + """ + + def __init__(self, path=None, text=None): + try: + self._text = self.VIEW_TEXT + except AttributeError: + if path is not None: + with open(path, 'r') as f: + self._text = f.read() + elif text is not None: + self._text = text + else: + self._text = "" + + if self._text[0] == "\n": + self._text = self._text[1:] + + newtext = self._text.lstrip() + amt = len(self._text) - len(newtext) + if (amt > 0): + base_indent = self._text[0:amt] + lines = self._text.splitlines() + newlines = [] + for line in lines: + if line.startswith(base_indent): + newlines.append(line[amt:]) + else: + newlines.append(line) + self._text = "\n".join(newlines) + + if self._text[-1] == "\n": + self._text = self._text[:-1] + + self._regentemplate = True + self._templatecache = None + + def __call__(self, model): + return self.template.render(**model) + + def __deepcopy__(self, memodict): + res = object.__new__(JinjaView) + res._text = copy.deepcopy(self._text, memodict) + + # regenerate the template on a deepcopy + res._regentemplate = True + res._templatecache = None + + return res + + @property + def template(self): + """Get the Compiled Template + + Gets the compiled template, using a cached copy if possible + (stored in attr:`_templatecache`) or otherwise recompiling + the template if the compiled template is not present or is + invalid (due to attr:`_regentemplate` being set to True). + + :returns: the compiled Jinja template + :rtype: :class:`jinja2.Template` + """ + + if self._templatecache is None or self._regentemplate: + self._templatecache = jinja2.Template(self._text) + self._regentemplate = False + + return self._templatecache + + def _gettext(self): + """Get the Template Text + + Gets the text of the current template + + :returns: the text of the Jinja template + :rtype: str + """ + + return self._text + + def _settext(self, textval): + """Set the Template Text + + Sets the text of the current template, marking it + for recompilation next time the compiled template + is retrived via attr:`template` . + + :param str textval: the new text of the Jinja template + """ + + self._text = textval + self.regentemplate = True + + text = property(_gettext, _settext) diff --git a/terracotta/openstack/common/report/views/json/__init__.py b/terracotta/openstack/common/report/views/json/__init__.py new file mode 100644 index 0000000..47bd33b --- /dev/null +++ b/terracotta/openstack/common/report/views/json/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides basic JSON views + +This module provides several basic views which serialize +models into JSON. +""" diff --git a/terracotta/openstack/common/report/views/json/generic.py b/terracotta/openstack/common/report/views/json/generic.py new file mode 100644 index 0000000..e5e1098 --- /dev/null +++ b/terracotta/openstack/common/report/views/json/generic.py @@ -0,0 +1,66 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides generic JSON views + +This modules defines several basic views for serializing +data to JSON. Submodels that have already been serialized +as JSON may have their string values marked with `__is_json__ += True` using :class:`openstack.common.report.utils.StringWithAttrs` +(each of the classes within this module does this automatically, +and non-naive serializers check for this attribute and handle +such strings specially) +""" + +import copy + +from oslo_serialization import jsonutils as json + +from nova.openstack.common.report import utils as utils + + +class BasicKeyValueView(object): + """A Basic Key-Value JSON View + + This view performs a naive serialization of a model + into JSON by simply calling :func:`json.dumps` on the model + """ + + def __call__(self, model): + res = utils.StringWithAttrs(json.dumps(model.data)) + res.__is_json__ = True + return res + + +class KeyValueView(object): + """A Key-Value JSON View + + This view performs advanced serialization to a model + into JSON. It does so by first checking all values to + see if they are marked as JSON. If so, they are deserialized + using :func:`json.loads`. Then, the copy of the model with all + JSON deserialized is reserialized into proper nested JSON using + :func:`json.dumps`. + """ + + def __call__(self, model): + # this part deals with subviews that were already serialized + cpy = copy.deepcopy(model) + for key in model.keys(): + if getattr(model[key], '__is_json__', False): + cpy[key] = json.loads(model[key]) + + res = utils.StringWithAttrs(json.dumps(cpy.data, sort_keys=True)) + res.__is_json__ = True + return res diff --git a/terracotta/openstack/common/report/views/text/__init__.py b/terracotta/openstack/common/report/views/text/__init__.py new file mode 100644 index 0000000..c097484 --- /dev/null +++ b/terracotta/openstack/common/report/views/text/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides basic text views + +This module provides several basic views which serialize +models into human-readable text. +""" diff --git a/terracotta/openstack/common/report/views/text/generic.py b/terracotta/openstack/common/report/views/text/generic.py new file mode 100644 index 0000000..3b30a07 --- /dev/null +++ b/terracotta/openstack/common/report/views/text/generic.py @@ -0,0 +1,202 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides generic text views + +This modules provides several generic views for +serializing models into human-readable text. +""" + +import collections as col + +import six + + +class MultiView(object): + """A Text View Containing Multiple Views + + This view simply serializes each + value in the data model, and then + joins them with newlines (ignoring + the key values altogether). This is + useful for serializing lists of models + (as array-like dicts). + """ + + def __call__(self, model): + res = [str(model[key]) for key in model] + return "\n".join(res) + + +class BasicKeyValueView(object): + """A Basic Key-Value Text View + + This view performs a naive serialization of a model into + text using a basic key-value method, where each + key-value pair is rendered as "key = str(value)" + """ + + def __call__(self, model): + res = "" + for key in model: + res += "{key} = {value}\n".format(key=key, value=model[key]) + + return res + + +class KeyValueView(object): + """A Key-Value Text View + + This view performs an advanced serialization of a model + into text by following the following set of rules: + + key : text + key = text + + rootkey : Mapping + :: + + rootkey = + serialize(key, value) + + key : Sequence + :: + + key = + serialize(item) + + :param str indent_str: the string used to represent one "indent" + :param str key_sep: the separator to use between keys and values + :param str dict_sep: the separator to use after a dictionary root key + :param str list_sep: the separator to use after a list root key + :param str anon_dict: the "key" to use when there is a dict in a list + (does not automatically use the dict separator) + :param before_dict: content to place on the line(s) before the a dict + root key (use None to avoid inserting an extra line) + :type before_dict: str or None + :param before_list: content to place on the line(s) before the a list + root key (use None to avoid inserting an extra line) + :type before_list: str or None + """ + + def __init__(self, + indent_str=' ', + key_sep=' = ', + dict_sep=' = ', + list_sep=' = ', + anon_dict='[dict]', + before_dict=None, + before_list=None): + self.indent_str = indent_str + self.key_sep = key_sep + self.dict_sep = dict_sep + self.list_sep = list_sep + self.anon_dict = anon_dict + self.before_dict = before_dict + self.before_list = before_list + + def __call__(self, model): + def serialize(root, rootkey, indent): + res = [] + if rootkey is not None: + res.append((self.indent_str * indent) + rootkey) + + if isinstance(root, col.Mapping): + if rootkey is None and indent > 0: + res.append((self.indent_str * indent) + self.anon_dict) + elif rootkey is not None: + res[0] += self.dict_sep + if self.before_dict is not None: + res.insert(0, self.before_dict) + + for key in sorted(root): + res.extend(serialize(root[key], key, indent + 1)) + elif (isinstance(root, col.Sequence) and + not isinstance(root, six.string_types)): + if rootkey is not None: + res[0] += self.list_sep + if self.before_list is not None: + res.insert(0, self.before_list) + + for val in sorted(root, key=str): + res.extend(serialize(val, None, indent + 1)) + else: + str_root = str(root) + if '\n' in str_root: + # we are in a submodel + if rootkey is not None: + res[0] += self.dict_sep + + list_root = [(self.indent_str * (indent + 1)) + line + for line in str_root.split('\n')] + res.extend(list_root) + else: + # just a normal key or list entry + try: + res[0] += self.key_sep + str_root + except IndexError: + res = [(self.indent_str * indent) + str_root] + + return res + + return "\n".join(serialize(model, None, -1)) + + +class TableView(object): + """A Basic Table Text View + + This view performs serialization of data into a basic table with + predefined column names and mappings. Column width is auto-calculated + evenly, column values are automatically truncated accordingly. Values + are centered in the columns. + + :param [str] column_names: the headers for each of the columns + :param [str] column_values: the item name to match each column to in + each row + :param str table_prop_name: the name of the property within the model + containing the row models + """ + + def __init__(self, column_names, column_values, table_prop_name): + self.table_prop_name = table_prop_name + self.column_names = column_names + self.column_values = column_values + self.column_width = (72 - len(column_names) + 1) // len(column_names) + + column_headers = "|".join( + "{ch[" + str(n) + "]: ^" + str(self.column_width) + "}" + for n in range(len(column_names)) + ) + + # correct for float-to-int roundoff error + test_fmt = column_headers.format(ch=column_names) + if len(test_fmt) < 72: + column_headers += ' ' * (72 - len(test_fmt)) + + vert_divider = '-' * 72 + self.header_fmt_str = column_headers + "\n" + vert_divider + "\n" + + self.row_fmt_str = "|".join( + "{cv[" + str(n) + "]: ^" + str(self.column_width) + "}" + for n in range(len(column_values)) + ) + + def __call__(self, model): + res = self.header_fmt_str.format(ch=self.column_names) + for raw_row in model[self.table_prop_name]: + row = [str(raw_row[prop_name]) for prop_name in self.column_values] + # double format is in case we have roundoff error + res += '{0: <72}\n'.format(self.row_fmt_str.format(cv=row)) + + return res diff --git a/terracotta/openstack/common/report/views/text/header.py b/terracotta/openstack/common/report/views/text/header.py new file mode 100644 index 0000000..58d06c0 --- /dev/null +++ b/terracotta/openstack/common/report/views/text/header.py @@ -0,0 +1,51 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Text Views With Headers + +This package defines several text views with headers +""" + + +class HeaderView(object): + """A Text View With a Header + + This view simply serializes the model and places the given + header on top. + + :param header: the header (can be anything on which str() can be called) + """ + + def __init__(self, header): + self.header = header + + def __call__(self, model): + return str(self.header) + "\n" + str(model) + + +class TitledView(HeaderView): + """A Text View With a Title + + This view simply serializes the model, and places + a preformatted header containing the given title + text on top. The title text can be up to 64 characters + long. + + :param str title: the title of the view + """ + + FORMAT_STR = ('=' * 72) + "\n===={0: ^64}====\n" + ('=' * 72) + + def __init__(self, title): + super(TitledView, self).__init__(self.FORMAT_STR.format(title)) diff --git a/terracotta/openstack/common/report/views/text/process.py b/terracotta/openstack/common/report/views/text/process.py new file mode 100644 index 0000000..0628ca9 --- /dev/null +++ b/terracotta/openstack/common/report/views/text/process.py @@ -0,0 +1,38 @@ +# Copyright 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides process view + +This module provides a view for +visualizing processes in human-readable formm +""" + +import nova.openstack.common.report.views.jinja_view as jv + + +class ProcessView(jv.JinjaView): + """A Process View + + This view displays process models defined by + :class:`openstack.common.report.models.process.ProcessModel` + """ + + VIEW_TEXT = ( + "Process {{ pid }} (under {{ parent_pid }}) " + "[ run by: {{ username }} ({{ uids.real|default('unknown uid') }})," + " state: {{ state }} ]\n" + "{% for child in children %}" + " {{ child }}" + "{% endfor %}" + ) diff --git a/terracotta/openstack/common/report/views/text/threading.py b/terracotta/openstack/common/report/views/text/threading.py new file mode 100644 index 0000000..18ef087 --- /dev/null +++ b/terracotta/openstack/common/report/views/text/threading.py @@ -0,0 +1,80 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides thread and stack-trace views + +This module provides a collection of views for +visualizing threads, green threads, and stack traces +in human-readable form. +""" + +from nova.openstack.common.report.views import jinja_view as jv + + +class StackTraceView(jv.JinjaView): + """A Stack Trace View + + This view displays stack trace models defined by + :class:`openstack.common.report.models.threading.StackTraceModel` + """ + + VIEW_TEXT = ( + "{% if root_exception is not none %}" + "Exception: {{ root_exception }}\n" + "------------------------------------\n" + "\n" + "{% endif %}" + "{% for line in lines %}\n" + "{{ line.filename }}:{{ line.line }} in {{ line.name }}\n" + " {% if line.code is not none %}" + "`{{ line.code }}`" + "{% else %}" + "(source not found)" + "{% endif %}\n" + "{% else %}\n" + "No Traceback!\n" + "{% endfor %}" + ) + + +class GreenThreadView(object): + """A Green Thread View + + This view displays a green thread provided by the data + model :class:`openstack.common.report.models.threading.GreenThreadModel` + """ + + FORMAT_STR = "------{thread_str: ^60}------" + "\n" + "{stack_trace}" + + def __call__(self, model): + return self.FORMAT_STR.format( + thread_str=" Green Thread ", + stack_trace=model.stack_trace + ) + + +class ThreadView(object): + """A Thread Collection View + + This view displays a python thread provided by the data + model :class:`openstack.common.report.models.threading.ThreadModel` # noqa + """ + + FORMAT_STR = "------{thread_str: ^60}------" + "\n" + "{stack_trace}" + + def __call__(self, model): + return self.FORMAT_STR.format( + thread_str=" Thread #{0} ".format(model.thread_id), + stack_trace=model.stack_trace + ) diff --git a/terracotta/openstack/common/report/views/xml/__init__.py b/terracotta/openstack/common/report/views/xml/__init__.py new file mode 100644 index 0000000..a40fec9 --- /dev/null +++ b/terracotta/openstack/common/report/views/xml/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides basic XML views + +This module provides several basic views which serialize +models into XML. +""" diff --git a/terracotta/openstack/common/report/views/xml/generic.py b/terracotta/openstack/common/report/views/xml/generic.py new file mode 100644 index 0000000..2a66708 --- /dev/null +++ b/terracotta/openstack/common/report/views/xml/generic.py @@ -0,0 +1,87 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides generic XML views + +This modules defines several basic views for serializing +data to XML. Submodels that have already been serialized +as XML may have their string values marked with `__is_xml__ += True` using :class:`openstack.common.report.utils.StringWithAttrs` +(each of the classes within this module does this automatically, +and non-naive serializers check for this attribute and handle +such strings specially) +""" + +import collections as col +import copy +import xml.etree.ElementTree as ET + +import six + +from nova.openstack.common.report import utils as utils + + +class KeyValueView(object): + """A Key-Value XML View + + This view performs advanced serialization of a data model + into XML. It first deserializes any values marked as XML so + that they can be properly reserialized later. It then follows + the following rules to perform serialization: + + key : text/xml + The tag name is the key name, and the contents are the text or xml + key : Sequence + A wrapper tag is created with the key name, and each item is placed + in an 'item' tag + key : Mapping + A wrapper tag is created with the key name, and the serialize is called + on each key-value pair (such that each key gets its own tag) + + :param str wrapper_name: the name of the top-level element + """ + + def __init__(self, wrapper_name="model"): + self.wrapper_name = wrapper_name + + def __call__(self, model): + # this part deals with subviews that were already serialized + cpy = copy.deepcopy(model) + for key, valstr in model.items(): + if getattr(valstr, '__is_xml__', False): + cpy[key] = ET.fromstring(valstr) + + def serialize(rootmodel, rootkeyname): + res = ET.Element(rootkeyname) + + if isinstance(rootmodel, col.Mapping): + for key in sorted(rootmodel): + res.append(serialize(rootmodel[key], key)) + elif (isinstance(rootmodel, col.Sequence) + and not isinstance(rootmodel, six.string_types)): + for val in sorted(rootmodel, key=str): + res.append(serialize(val, 'item')) + elif ET.iselement(rootmodel): + res.append(rootmodel) + else: + res.text = str(rootmodel) + + return res + + str_ = ET.tostring(serialize(cpy, + self.wrapper_name), + encoding="utf-8").decode("utf-8") + res = utils.StringWithAttrs(str_) + res.__is_xml__ = True + return res diff --git a/terracotta/openstack/common/service.py b/terracotta/openstack/common/service.py new file mode 100644 index 0000000..58aaa79 --- /dev/null +++ b/terracotta/openstack/common/service.py @@ -0,0 +1,509 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Generic Node base class for all workers that run on hosts.""" + +import errno +import logging +import os +import random +import signal +import sys +import time + +try: + # Importing just the symbol here because the io module does not + # exist in Python 2.6. + from io import UnsupportedOperation # noqa +except ImportError: + # Python 2.6 + UnsupportedOperation = None + +import eventlet +from eventlet import event +from oslo_config import cfg + +from nova.openstack.common import eventlet_backdoor +from nova.openstack.common._i18n import _LE, _LI, _LW +from nova.openstack.common import systemd +from nova.openstack.common import threadgroup + + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +def _sighup_supported(): + return hasattr(signal, 'SIGHUP') + + +def _is_daemon(): + # The process group for a foreground process will match the + # process group of the controlling terminal. If those values do + # not match, or ioctl() fails on the stdout file handle, we assume + # the process is running in the background as a daemon. + # http://www.gnu.org/software/bash/manual/bashref.html#Job-Control-Basics + try: + is_daemon = os.getpgrp() != os.tcgetpgrp(sys.stdout.fileno()) + except OSError as err: + if err.errno == errno.ENOTTY: + # Assume we are a daemon because there is no terminal. + is_daemon = True + else: + raise + except UnsupportedOperation: + # Could not get the fileno for stdout, so we must be a daemon. + is_daemon = True + return is_daemon + + +def _is_sighup_and_daemon(signo): + if not (_sighup_supported() and signo == signal.SIGHUP): + # Avoid checking if we are a daemon, because the signal isn't + # SIGHUP. + return False + return _is_daemon() + + +def _signo_to_signame(signo): + signals = {signal.SIGTERM: 'SIGTERM', + signal.SIGINT: 'SIGINT'} + if _sighup_supported(): + signals[signal.SIGHUP] = 'SIGHUP' + return signals[signo] + + +def _set_signals_handler(handler): + signal.signal(signal.SIGTERM, handler) + signal.signal(signal.SIGINT, handler) + if _sighup_supported(): + signal.signal(signal.SIGHUP, handler) + + +class Launcher(object): + """Launch one or more services and wait for them to complete.""" + + def __init__(self): + """Initialize the service launcher. + + :returns: None + + """ + self.services = Services() + self.backdoor_port = eventlet_backdoor.initialize_if_enabled() + + def launch_service(self, service): + """Load and start the given service. + + :param service: The service you would like to start. + :returns: None + + """ + service.backdoor_port = self.backdoor_port + self.services.add(service) + + def stop(self): + """Stop all services which are currently running. + + :returns: None + + """ + self.services.stop() + + def wait(self): + """Waits until all services have been stopped, and then returns. + + :returns: None + + """ + self.services.wait() + + def restart(self): + """Reload config files and restart service. + + :returns: None + + """ + cfg.CONF.reload_config_files() + self.services.restart() + + +class SignalExit(SystemExit): + def __init__(self, signo, exccode=1): + super(SignalExit, self).__init__(exccode) + self.signo = signo + + +class ServiceLauncher(Launcher): + def _handle_signal(self, signo, frame): + # Allow the process to be killed again and die from natural causes + _set_signals_handler(signal.SIG_DFL) + raise SignalExit(signo) + + def handle_signal(self): + _set_signals_handler(self._handle_signal) + + def _wait_for_exit_or_signal(self, ready_callback=None): + status = None + signo = 0 + + LOG.debug('Full set of CONF:') + CONF.log_opt_values(LOG, logging.DEBUG) + + try: + if ready_callback: + ready_callback() + super(ServiceLauncher, self).wait() + except SignalExit as exc: + signame = _signo_to_signame(exc.signo) + LOG.info(_LI('Caught %s, exiting'), signame) + status = exc.code + signo = exc.signo + except SystemExit as exc: + status = exc.code + finally: + self.stop() + + return status, signo + + def wait(self, ready_callback=None): + systemd.notify_once() + while True: + self.handle_signal() + status, signo = self._wait_for_exit_or_signal(ready_callback) + if not _is_sighup_and_daemon(signo): + return status + self.restart() + + +class ServiceWrapper(object): + def __init__(self, service, workers): + self.service = service + self.workers = workers + self.children = set() + self.forktimes = [] + + +class ProcessLauncher(object): + _signal_handlers_set = set() + + @classmethod + def _handle_class_signals(cls, *args, **kwargs): + for handler in cls._signal_handlers_set: + handler(*args, **kwargs) + + def __init__(self): + """Constructor.""" + + self.children = {} + self.sigcaught = None + self.running = True + rfd, self.writepipe = os.pipe() + self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') + self.handle_signal() + + def handle_signal(self): + self._signal_handlers_set.add(self._handle_signal) + _set_signals_handler(self._handle_class_signals) + + def _handle_signal(self, signo, frame): + self.sigcaught = signo + self.running = False + + # Allow the process to be killed again and die from natural causes + _set_signals_handler(signal.SIG_DFL) + + def _pipe_watcher(self): + # This will block until the write end is closed when the parent + # dies unexpectedly + self.readpipe.read() + + LOG.info(_LI('Parent process has died unexpectedly, exiting')) + + sys.exit(1) + + def _child_process_handle_signal(self): + # Setup child signal handlers differently + def _sigterm(*args): + signal.signal(signal.SIGTERM, signal.SIG_DFL) + raise SignalExit(signal.SIGTERM) + + def _sighup(*args): + signal.signal(signal.SIGHUP, signal.SIG_DFL) + raise SignalExit(signal.SIGHUP) + + signal.signal(signal.SIGTERM, _sigterm) + if _sighup_supported(): + signal.signal(signal.SIGHUP, _sighup) + # Block SIGINT and let the parent send us a SIGTERM + signal.signal(signal.SIGINT, signal.SIG_IGN) + + def _child_wait_for_exit_or_signal(self, launcher): + status = 0 + signo = 0 + + # NOTE(johannes): All exceptions are caught to ensure this + # doesn't fallback into the loop spawning children. It would + # be bad for a child to spawn more children. + try: + launcher.wait() + except SignalExit as exc: + signame = _signo_to_signame(exc.signo) + LOG.info(_LI('Child caught %s, exiting'), signame) + status = exc.code + signo = exc.signo + except SystemExit as exc: + status = exc.code + except BaseException: + LOG.exception(_LE('Unhandled exception')) + status = 2 + finally: + launcher.stop() + + return status, signo + + def _child_process(self, service): + self._child_process_handle_signal() + + # Reopen the eventlet hub to make sure we don't share an epoll + # fd with parent and/or siblings, which would be bad + eventlet.hubs.use_hub() + + # Close write to ensure only parent has it open + os.close(self.writepipe) + # Create greenthread to watch for parent to close pipe + eventlet.spawn_n(self._pipe_watcher) + + # Reseed random number generator + random.seed() + + launcher = Launcher() + launcher.launch_service(service) + return launcher + + def _start_child(self, wrap): + if len(wrap.forktimes) > wrap.workers: + # Limit ourselves to one process a second (over the period of + # number of workers * 1 second). This will allow workers to + # start up quickly but ensure we don't fork off children that + # die instantly too quickly. + if time.time() - wrap.forktimes[0] < wrap.workers: + LOG.info(_LI('Forking too fast, sleeping')) + time.sleep(1) + + wrap.forktimes.pop(0) + + wrap.forktimes.append(time.time()) + + pid = os.fork() + if pid == 0: + launcher = self._child_process(wrap.service) + while True: + self._child_process_handle_signal() + status, signo = self._child_wait_for_exit_or_signal(launcher) + if not _is_sighup_and_daemon(signo): + break + launcher.restart() + + os._exit(status) + + LOG.info(_LI('Started child %d'), pid) + + wrap.children.add(pid) + self.children[pid] = wrap + + return pid + + def launch_service(self, service, workers=1): + wrap = ServiceWrapper(service, workers) + + LOG.info(_LI('Starting %d workers'), wrap.workers) + while self.running and len(wrap.children) < wrap.workers: + self._start_child(wrap) + + def _wait_child(self): + try: + # Block while any of child processes have exited + pid, status = os.waitpid(0, 0) + if not pid: + return None + except OSError as exc: + if exc.errno not in (errno.EINTR, errno.ECHILD): + raise + return None + + if os.WIFSIGNALED(status): + sig = os.WTERMSIG(status) + LOG.info(_LI('Child %(pid)d killed by signal %(sig)d'), + dict(pid=pid, sig=sig)) + else: + code = os.WEXITSTATUS(status) + LOG.info(_LI('Child %(pid)s exited with status %(code)d'), + dict(pid=pid, code=code)) + + if pid not in self.children: + LOG.warning(_LW('pid %d not in child list'), pid) + return None + + wrap = self.children.pop(pid) + wrap.children.remove(pid) + return wrap + + def _respawn_children(self): + while self.running: + wrap = self._wait_child() + if not wrap: + continue + while self.running and len(wrap.children) < wrap.workers: + self._start_child(wrap) + + def wait(self): + """Loop waiting on children to die and respawning as necessary.""" + + systemd.notify_once() + LOG.debug('Full set of CONF:') + CONF.log_opt_values(LOG, logging.DEBUG) + + try: + while True: + self.handle_signal() + self._respawn_children() + # No signal means that stop was called. Don't clean up here. + if not self.sigcaught: + return + + signame = _signo_to_signame(self.sigcaught) + LOG.info(_LI('Caught %s, stopping children'), signame) + if not _is_sighup_and_daemon(self.sigcaught): + break + + cfg.CONF.reload_config_files() + for service in set( + [wrap.service for wrap in self.children.values()]): + service.reset() + + for pid in self.children: + os.kill(pid, signal.SIGHUP) + + self.running = True + self.sigcaught = None + except eventlet.greenlet.GreenletExit: + LOG.info(_LI("Wait called after thread killed. Cleaning up.")) + + self.stop() + + def stop(self): + """Terminate child processes and wait on each.""" + self.running = False + for pid in self.children: + try: + os.kill(pid, signal.SIGTERM) + except OSError as exc: + if exc.errno != errno.ESRCH: + raise + + # Wait for children to die + if self.children: + LOG.info(_LI('Waiting on %d children to exit'), len(self.children)) + while self.children: + self._wait_child() + + +class Service(object): + """Service object for binaries running on hosts.""" + + def __init__(self, threads=1000): + self.tg = threadgroup.ThreadGroup(threads) + + # signal that the service is done shutting itself down: + self._done = event.Event() + + def reset(self): + # NOTE(Fengqian): docs for Event.reset() recommend against using it + self._done = event.Event() + + def start(self): + pass + + def stop(self, graceful=False): + self.tg.stop(graceful) + self.tg.wait() + # Signal that service cleanup is done: + if not self._done.ready(): + self._done.send() + + def wait(self): + self._done.wait() + + +class Services(object): + + def __init__(self): + self.services = [] + self.tg = threadgroup.ThreadGroup() + self.done = event.Event() + + def add(self, service): + self.services.append(service) + self.tg.add_thread(self.run_service, service, self.done) + + def stop(self): + # wait for graceful shutdown of services: + for service in self.services: + service.stop() + service.wait() + + # Each service has performed cleanup, now signal that the run_service + # wrapper threads can now die: + if not self.done.ready(): + self.done.send() + + # reap threads: + self.tg.stop() + + def wait(self): + self.tg.wait() + + def restart(self): + self.stop() + self.done = event.Event() + for restart_service in self.services: + restart_service.reset() + self.tg.add_thread(self.run_service, restart_service, self.done) + + @staticmethod + def run_service(service, done): + """Service start wrapper. + + :param service: service to run + :param done: event to wait on until a shutdown is triggered + :returns: None + + """ + service.start() + done.wait() + + +def launch(service, workers=1): + if workers is None or workers == 1: + launcher = ServiceLauncher() + launcher.launch_service(service) + else: + launcher = ProcessLauncher() + launcher.launch_service(service, workers=workers) + + return launcher diff --git a/terracotta/openstack/common/sslutils.py b/terracotta/openstack/common/sslutils.py new file mode 100644 index 0000000..86c6f0a --- /dev/null +++ b/terracotta/openstack/common/sslutils.py @@ -0,0 +1,81 @@ +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import os +import ssl + +from oslo_config import cfg + +from nova.openstack.common._i18n import _ + + +ssl_opts = [ + cfg.StrOpt('ca_file', + help="CA certificate file to use to verify " + "connecting clients."), + cfg.StrOpt('cert_file', + help="Certificate file to use when starting " + "the server securely."), + cfg.StrOpt('key_file', + help="Private key file to use when starting " + "the server securely."), +] + +CONF = cfg.CONF +config_section = 'ssl' +CONF.register_opts(ssl_opts, config_section) + + +def list_opts(): + """Entry point for oslo-config-generator.""" + return [(config_section, copy.deepcopy(ssl_opts))] + + +def is_enabled(): + cert_file = CONF.ssl.cert_file + key_file = CONF.ssl.key_file + ca_file = CONF.ssl.ca_file + use_ssl = cert_file or key_file + + if cert_file and not os.path.exists(cert_file): + raise RuntimeError(_("Unable to find cert_file : %s") % cert_file) + + if ca_file and not os.path.exists(ca_file): + raise RuntimeError(_("Unable to find ca_file : %s") % ca_file) + + if key_file and not os.path.exists(key_file): + raise RuntimeError(_("Unable to find key_file : %s") % key_file) + + if use_ssl and (not cert_file or not key_file): + raise RuntimeError(_("When running server in SSL mode, you must " + "specify both a cert_file and key_file " + "option value in your configuration file")) + + return use_ssl + + +def wrap(sock): + ssl_kwargs = { + 'server_side': True, + 'certfile': CONF.ssl.cert_file, + 'keyfile': CONF.ssl.key_file, + 'cert_reqs': ssl.CERT_NONE, + } + + if CONF.ssl.ca_file: + ssl_kwargs['ca_certs'] = CONF.ssl.ca_file + ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED + + return ssl.wrap_socket(sock, **ssl_kwargs) diff --git a/terracotta/openstack/common/systemd.py b/terracotta/openstack/common/systemd.py new file mode 100644 index 0000000..36243b3 --- /dev/null +++ b/terracotta/openstack/common/systemd.py @@ -0,0 +1,105 @@ +# Copyright 2012-2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Helper module for systemd service readiness notification. +""" + +import logging +import os +import socket +import sys + + +LOG = logging.getLogger(__name__) + + +def _abstractify(socket_name): + if socket_name.startswith('@'): + # abstract namespace socket + socket_name = '\0%s' % socket_name[1:] + return socket_name + + +def _sd_notify(unset_env, msg): + notify_socket = os.getenv('NOTIFY_SOCKET') + if notify_socket: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + try: + sock.connect(_abstractify(notify_socket)) + sock.sendall(msg) + if unset_env: + del os.environ['NOTIFY_SOCKET'] + except EnvironmentError: + LOG.debug("Systemd notification failed", exc_info=True) + finally: + sock.close() + + +def notify(): + """Send notification to Systemd that service is ready. + + For details see + http://www.freedesktop.org/software/systemd/man/sd_notify.html + """ + _sd_notify(False, 'READY=1') + + +def notify_once(): + """Send notification once to Systemd that service is ready. + + Systemd sets NOTIFY_SOCKET environment variable with the name of the + socket listening for notifications from services. + This method removes the NOTIFY_SOCKET environment variable to ensure + notification is sent only once. + """ + _sd_notify(True, 'READY=1') + + +def onready(notify_socket, timeout): + """Wait for systemd style notification on the socket. + + :param notify_socket: local socket address + :type notify_socket: string + :param timeout: socket timeout + :type timeout: float + :returns: 0 service ready + 1 service not ready + 2 timeout occurred + """ + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + sock.settimeout(timeout) + sock.bind(_abstractify(notify_socket)) + try: + msg = sock.recv(512) + except socket.timeout: + return 2 + finally: + sock.close() + if 'READY=1' in msg: + return 0 + else: + return 1 + + +if __name__ == '__main__': + # simple CLI for testing + if len(sys.argv) == 1: + notify() + elif len(sys.argv) >= 2: + timeout = float(sys.argv[1]) + notify_socket = os.getenv('NOTIFY_SOCKET') + if notify_socket: + retval = onready(notify_socket, timeout) + sys.exit(retval) diff --git a/terracotta/openstack/common/threadgroup.py b/terracotta/openstack/common/threadgroup.py new file mode 100644 index 0000000..6879965 --- /dev/null +++ b/terracotta/openstack/common/threadgroup.py @@ -0,0 +1,149 @@ +# Copyright 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import logging +import threading + +import eventlet +from eventlet import greenpool + +from nova.openstack.common import loopingcall + + +LOG = logging.getLogger(__name__) + + +def _thread_done(gt, *args, **kwargs): + """Callback function to be passed to GreenThread.link() when we spawn() + Calls the :class:`ThreadGroup` to notify if. + + """ + kwargs['group'].thread_done(kwargs['thread']) + + +class Thread(object): + """Wrapper around a greenthread, that holds a reference to the + :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when + it has done so it can be removed from the threads list. + """ + def __init__(self, thread, group): + self.thread = thread + self.thread.link(_thread_done, group=group, thread=self) + + def stop(self): + self.thread.kill() + + def wait(self): + return self.thread.wait() + + def link(self, func, *args, **kwargs): + self.thread.link(func, *args, **kwargs) + + +class ThreadGroup(object): + """The point of the ThreadGroup class is to: + + * keep track of timers and greenthreads (making it easier to stop them + when need be). + * provide an easy API to add timers. + """ + def __init__(self, thread_pool_size=10): + self.pool = greenpool.GreenPool(thread_pool_size) + self.threads = [] + self.timers = [] + + def add_dynamic_timer(self, callback, initial_delay=None, + periodic_interval_max=None, *args, **kwargs): + timer = loopingcall.DynamicLoopingCall(callback, *args, **kwargs) + timer.start(initial_delay=initial_delay, + periodic_interval_max=periodic_interval_max) + self.timers.append(timer) + + def add_timer(self, interval, callback, initial_delay=None, + *args, **kwargs): + pulse = loopingcall.FixedIntervalLoopingCall(callback, *args, **kwargs) + pulse.start(interval=interval, + initial_delay=initial_delay) + self.timers.append(pulse) + + def add_thread(self, callback, *args, **kwargs): + gt = self.pool.spawn(callback, *args, **kwargs) + th = Thread(gt, self) + self.threads.append(th) + return th + + def thread_done(self, thread): + self.threads.remove(thread) + + def _stop_threads(self): + current = threading.current_thread() + + # Iterate over a copy of self.threads so thread_done doesn't + # modify the list while we're iterating + for x in self.threads[:]: + if x is current: + # don't kill the current thread. + continue + try: + x.stop() + except eventlet.greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) + + def stop_timers(self): + for x in self.timers: + try: + x.stop() + except Exception as ex: + LOG.exception(ex) + self.timers = [] + + def stop(self, graceful=False): + """stop function has the option of graceful=True/False. + + * In case of graceful=True, wait for all threads to be finished. + Never kill threads. + * In case of graceful=False, kill threads immediately. + """ + self.stop_timers() + if graceful: + # In case of graceful=True, wait for all threads to be + # finished, never kill threads + self.wait() + else: + # In case of graceful=False(Default), kill threads + # immediately + self._stop_threads() + + def wait(self): + for x in self.timers: + try: + x.wait() + except eventlet.greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) + current = threading.current_thread() + + # Iterate over a copy of self.threads so thread_done doesn't + # modify the list while we're iterating + for x in self.threads[:]: + if x is current: + continue + try: + x.wait() + except eventlet.greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) diff --git a/terracotta/openstack/common/versionutils.py b/terracotta/openstack/common/versionutils.py new file mode 100644 index 0000000..30f92b3 --- /dev/null +++ b/terracotta/openstack/common/versionutils.py @@ -0,0 +1,262 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Helpers for comparing version strings. +""" + +import copy +import functools +import inspect +import logging + +from oslo_config import cfg +import pkg_resources +import six + +from nova.openstack.common._i18n import _ + + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +deprecated_opts = [ + cfg.BoolOpt('fatal_deprecations', + default=False, + help='Enables or disables fatal status of deprecations.'), +] + + +def list_opts(): + """Entry point for oslo.config-generator. + """ + return [(None, copy.deepcopy(deprecated_opts))] + + +class deprecated(object): + """A decorator to mark callables as deprecated. + + This decorator logs a deprecation message when the callable it decorates is + used. The message will include the release where the callable was + deprecated, the release where it may be removed and possibly an optional + replacement. + + Examples: + + 1. Specifying the required deprecated release + + >>> @deprecated(as_of=deprecated.ICEHOUSE) + ... def a(): pass + + 2. Specifying a replacement: + + >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()') + ... def b(): pass + + 3. Specifying the release where the functionality may be removed: + + >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1) + ... def c(): pass + + 4. Specifying the deprecated functionality will not be removed: + >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=0) + ... def d(): pass + + 5. Specifying a replacement, deprecated functionality will not be removed: + >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()', remove_in=0) + ... def e(): pass + + """ + + # NOTE(morganfainberg): Bexar is used for unit test purposes, it is + # expected we maintain a gap between Bexar and Folsom in this list. + BEXAR = 'B' + FOLSOM = 'F' + GRIZZLY = 'G' + HAVANA = 'H' + ICEHOUSE = 'I' + JUNO = 'J' + KILO = 'K' + LIBERTY = 'L' + + _RELEASES = { + # NOTE(morganfainberg): Bexar is used for unit test purposes, it is + # expected we maintain a gap between Bexar and Folsom in this list. + 'B': 'Bexar', + 'F': 'Folsom', + 'G': 'Grizzly', + 'H': 'Havana', + 'I': 'Icehouse', + 'J': 'Juno', + 'K': 'Kilo', + 'L': 'Liberty', + } + + _deprecated_msg_with_alternative = _( + '%(what)s is deprecated as of %(as_of)s in favor of ' + '%(in_favor_of)s and may be removed in %(remove_in)s.') + + _deprecated_msg_no_alternative = _( + '%(what)s is deprecated as of %(as_of)s and may be ' + 'removed in %(remove_in)s. It will not be superseded.') + + _deprecated_msg_with_alternative_no_removal = _( + '%(what)s is deprecated as of %(as_of)s in favor of %(in_favor_of)s.') + + _deprecated_msg_with_no_alternative_no_removal = _( + '%(what)s is deprecated as of %(as_of)s. It will not be superseded.') + + def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None): + """Initialize decorator + + :param as_of: the release deprecating the callable. Constants + are define in this class for convenience. + :param in_favor_of: the replacement for the callable (optional) + :param remove_in: an integer specifying how many releases to wait + before removing (default: 2) + :param what: name of the thing being deprecated (default: the + callable's name) + + """ + self.as_of = as_of + self.in_favor_of = in_favor_of + self.remove_in = remove_in + self.what = what + + def __call__(self, func_or_cls): + if not self.what: + self.what = func_or_cls.__name__ + '()' + msg, details = self._build_message() + + if inspect.isfunction(func_or_cls): + + @six.wraps(func_or_cls) + def wrapped(*args, **kwargs): + report_deprecated_feature(LOG, msg, details) + return func_or_cls(*args, **kwargs) + return wrapped + elif inspect.isclass(func_or_cls): + orig_init = func_or_cls.__init__ + + # TODO(tsufiev): change `functools` module to `six` as + # soon as six 1.7.4 (with fix for passing `assigned` + # argument to underlying `functools.wraps`) is released + # and added to the oslo-incubator requrements + @functools.wraps(orig_init, assigned=('__name__', '__doc__')) + def new_init(self, *args, **kwargs): + report_deprecated_feature(LOG, msg, details) + orig_init(self, *args, **kwargs) + func_or_cls.__init__ = new_init + return func_or_cls + else: + raise TypeError('deprecated can be used only with functions or ' + 'classes') + + def _get_safe_to_remove_release(self, release): + # TODO(dstanek): this method will have to be reimplemented once + # when we get to the X release because once we get to the Y + # release, what is Y+2? + new_release = chr(ord(release) + self.remove_in) + if new_release in self._RELEASES: + return self._RELEASES[new_release] + else: + return new_release + + def _build_message(self): + details = dict(what=self.what, + as_of=self._RELEASES[self.as_of], + remove_in=self._get_safe_to_remove_release(self.as_of)) + + if self.in_favor_of: + details['in_favor_of'] = self.in_favor_of + if self.remove_in > 0: + msg = self._deprecated_msg_with_alternative + else: + # There are no plans to remove this function, but it is + # now deprecated. + msg = self._deprecated_msg_with_alternative_no_removal + else: + if self.remove_in > 0: + msg = self._deprecated_msg_no_alternative + else: + # There are no plans to remove this function, but it is + # now deprecated. + msg = self._deprecated_msg_with_no_alternative_no_removal + return msg, details + + +def is_compatible(requested_version, current_version, same_major=True): + """Determine whether `requested_version` is satisfied by + `current_version`; in other words, `current_version` is >= + `requested_version`. + + :param requested_version: version to check for compatibility + :param current_version: version to check against + :param same_major: if True, the major version must be identical between + `requested_version` and `current_version`. This is used when a + major-version difference indicates incompatibility between the two + versions. Since this is the common-case in practice, the default is + True. + :returns: True if compatible, False if not + """ + requested_parts = pkg_resources.parse_version(requested_version) + current_parts = pkg_resources.parse_version(current_version) + + if same_major and (requested_parts[0] != current_parts[0]): + return False + + return current_parts >= requested_parts + + +# Track the messages we have sent already. See +# report_deprecated_feature(). +_deprecated_messages_sent = {} + + +def report_deprecated_feature(logger, msg, *args, **kwargs): + """Call this function when a deprecated feature is used. + + If the system is configured for fatal deprecations then the message + is logged at the 'critical' level and :class:`DeprecatedConfig` will + be raised. + + Otherwise, the message will be logged (once) at the 'warn' level. + + :raises: :class:`DeprecatedConfig` if the system is configured for + fatal deprecations. + """ + stdmsg = _("Deprecated: %s") % msg + CONF.register_opts(deprecated_opts) + if CONF.fatal_deprecations: + logger.critical(stdmsg, *args, **kwargs) + raise DeprecatedConfig(msg=stdmsg) + + # Using a list because a tuple with dict can't be stored in a set. + sent_args = _deprecated_messages_sent.setdefault(msg, list()) + + if args in sent_args: + # Already logged this message, so don't log it again. + return + + sent_args.append(args) + logger.warn(stdmsg, *args, **kwargs) + + +class DeprecatedConfig(Exception): + message = _("Fatal call to deprecated config: %(msg)s") + + def __init__(self, msg): + super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/terracotta/rpc.py b/terracotta/rpc.py new file mode 100644 index 0000000..6341859 --- /dev/null +++ b/terracotta/rpc.py @@ -0,0 +1,395 @@ +# Copyright 2014 - Mirantis, Inc. +# Copyright 2015 - StackStorm, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.config import cfg +from oslo import messaging +from oslo_messaging.rpc import client + +from mistral import context as auth_ctx +from mistral.engine import base +from mistral import exceptions as exc +from mistral.openstack.common import log as logging +from mistral.workflow import utils as wf_utils + +LOG = logging.getLogger(__name__) + + +_TRANSPORT = None + +_ENGINE_CLIENT = None +_EXECUTOR_CLIENT = None + + +def cleanup(): + """Intended to be used by tests to recreate all RPC related objects.""" + + global _TRANSPORT + global _ENGINE_CLIENT + global _EXECUTOR_CLIENT + + _TRANSPORT = None + _ENGINE_CLIENT = None + _EXECUTOR_CLIENT = None + + +def get_transport(): + global _TRANSPORT + + if not _TRANSPORT: + _TRANSPORT = messaging.get_transport(cfg.CONF) + + return _TRANSPORT + + +def get_engine_client(): + global _ENGINE_CLIENT + + if not _ENGINE_CLIENT: + _ENGINE_CLIENT = EngineClient(get_transport()) + + return _ENGINE_CLIENT + + +def get_executor_client(): + global _EXECUTOR_CLIENT + + if not _EXECUTOR_CLIENT: + _EXECUTOR_CLIENT = ExecutorClient(get_transport()) + + return _EXECUTOR_CLIENT + + +class EngineServer(object): + """RPC Engine server.""" + + def __init__(self, engine): + self._engine = engine + + def start_workflow(self, rpc_ctx, workflow_name, workflow_input, params): + """Receives calls over RPC to start workflows on engine. + + :param rpc_ctx: RPC request context. + :return: Workflow execution. + """ + + LOG.info( + "Received RPC request 'start_workflow'[rpc_ctx=%s," + " workflow_name=%s, workflow_input=%s, params=%s]" + % (rpc_ctx, workflow_name, workflow_input, params) + ) + + return self._engine.start_workflow( + workflow_name, + workflow_input, + **params + ) + + def on_task_state_change(self, rpc_ctx, task_ex_id, state): + return self._engine.on_task_state_change(task_ex_id, state) + + def on_action_complete(self, rpc_ctx, action_ex_id, result_data, + result_error): + """Receives RPC calls to communicate action result to engine. + + :param rpc_ctx: RPC request context. + :param action_ex_id: Action execution id. + :return: Action execution. + """ + + result = wf_utils.Result(result_data, result_error) + + LOG.info( + "Received RPC request 'on_action_complete'[rpc_ctx=%s," + " action_ex_id=%s, result=%s]" % (rpc_ctx, action_ex_id, result) + ) + + return self._engine.on_action_complete(action_ex_id, result) + + def pause_workflow(self, rpc_ctx, execution_id): + """Receives calls over RPC to pause workflows on engine. + + :param rpc_ctx: Request context. + :return: Workflow execution. + """ + + LOG.info( + "Received RPC request 'pause_workflow'[rpc_ctx=%s," + " execution_id=%s]" % (rpc_ctx, execution_id) + ) + + return self._engine.pause_workflow(execution_id) + + def resume_workflow(self, rpc_ctx, execution_id): + """Receives calls over RPC to resume workflows on engine. + + :param rpc_ctx: RPC request context. + :return: Workflow execution. + """ + + LOG.info( + "Received RPC request 'resume_workflow'[rpc_ctx=%s," + " execution_id=%s]" % (rpc_ctx, execution_id) + ) + + return self._engine.resume_workflow(execution_id) + + def stop_workflow(self, rpc_ctx, execution_id, state, message=None): + """Receives calls over RPC to stop workflows on engine. + + Sets execution state to SUCCESS or ERROR. No more tasks will be + scheduled. Running tasks won't be killed, but their results + will be ignored. + + :param rpc_ctx: RPC request context. + :param execution_id: Workflow execution id. + :param state: State assigned to the workflow. Permitted states are + SUCCESS or ERROR. + :param message: Optional information string. + + :return: Workflow execution. + """ + + LOG.info( + "Received RPC request 'stop_workflow'[rpc_ctx=%s, execution_id=%s," + " state=%s, message=%s]" % (rpc_ctx, execution_id, state, message) + ) + + return self._engine.stop_workflow(execution_id, state, message) + + def rollback_workflow(self, rpc_ctx, execution_id): + """Receives calls over RPC to rollback workflows on engine. + + :param rpc_ctx: RPC request context. + :return: Workflow execution. + """ + + LOG.info( + "Received RPC request 'rollback_workflow'[rpc_ctx=%s," + " execution_id=%s]" % (rpc_ctx, execution_id) + ) + + return self._engine.resume_workflow(execution_id) + + +def wrap_messaging_exception(method): + """This decorator unwrap remote error in one of MistralException. + + oslo.messaging has different behavior on raising exceptions + when fake or rabbit transports are used. In case of rabbit + transport it raises wrapped RemoteError which forwards directly + to API. Wrapped RemoteError contains one of MistralException raised + remotely on Engine and for correct exception interpretation we + need to unwrap and raise given exception and manually send it to + API layer. + """ + def decorator(*args, **kwargs): + try: + return method(*args, **kwargs) + + except client.RemoteError as e: + exc_cls = getattr(exc, e.exc_type) + raise exc_cls(e.value) + + return decorator + + +class EngineClient(base.Engine): + """RPC Engine client.""" + + def __init__(self, transport): + """Constructs an RPC client for engine. + + :param transport: Messaging transport. + """ + serializer = auth_ctx.RpcContextSerializer( + auth_ctx.JsonPayloadSerializer()) + + self._client = messaging.RPCClient( + transport, + messaging.Target(topic=cfg.CONF.engine.topic), + serializer=serializer + ) + + @wrap_messaging_exception + def start_workflow(self, wf_name, wf_input, **params): + """Starts workflow sending a request to engine over RPC. + + :return: Workflow execution. + """ + return self._client.call( + auth_ctx.ctx(), + 'start_workflow', + workflow_name=wf_name, + workflow_input=wf_input or {}, + params=params + ) + + def on_task_state_change(self, task_ex_id, state): + return self._client.call( + auth_ctx.ctx(), + 'on_task_state_change', + task_ex_id=task_ex_id, + state=state + ) + + @wrap_messaging_exception + def on_action_complete(self, action_ex_id, result): + """Conveys action result to Mistral Engine. + + This method should be used by clients of Mistral Engine to update + state of a action execution once action has executed. One of the + clients of this method is Mistral REST API server that receives + action result from the outside action handlers. + + Note: calling this method serves an event notifying Mistral that + it possibly needs to move the workflow on, i.e. run other workflow + tasks for which all dependencies are satisfied. + + :return: Task. + """ + + return self._client.call( + auth_ctx.ctx(), + 'on_action_complete', + action_ex_id=action_ex_id, + result_data=result.data, + result_error=result.error + ) + + @wrap_messaging_exception + def pause_workflow(self, execution_id): + """Stops the workflow with the given execution id. + + :return: Workflow execution. + """ + + return self._client.call( + auth_ctx.ctx(), + 'pause_workflow', + execution_id=execution_id + ) + + @wrap_messaging_exception + def resume_workflow(self, execution_id): + """Resumes the workflow with the given execution id. + + :return: Workflow execution. + """ + + return self._client.call( + auth_ctx.ctx(), + 'resume_workflow', + execution_id=execution_id + ) + + @wrap_messaging_exception + def stop_workflow(self, execution_id, state, message=None): + """Stops workflow execution with given status. + + Once stopped, the workflow is complete with SUCCESS or ERROR, + and can not be resumed. + + :param execution_id: Workflow execution id + :param state: State assigned to the workflow: SUCCESS or ERROR + :param message: Optional information string + + :return: Workflow execution, model.Execution + """ + + return self._client.call( + auth_ctx.ctx(), + 'stop_workflow', + execution_id=execution_id, + state=state, + message=message + ) + + @wrap_messaging_exception + def rollback_workflow(self, execution_id): + """Rolls back the workflow with the given execution id. + + :return: Workflow execution. + """ + + return self._client.call( + auth_ctx.ctx(), + 'rollback_workflow', + execution_id=execution_id + ) + + +class ExecutorServer(object): + """RPC Executor server.""" + + def __init__(self, executor): + self._executor = executor + + def run_action(self, rpc_ctx, action_ex_id, action_class_str, + attributes, params): + """Receives calls over RPC to run action on executor. + + :param rpc_ctx: RPC request context dictionary. + """ + + LOG.info( + "Received RPC request 'run_action'[rpc_ctx=%s," + " action_ex_id=%s, action_class=%s, attributes=%s, params=%s]" + % (rpc_ctx, action_ex_id, action_class_str, attributes, params) + ) + + self._executor.run_action( + action_ex_id, + action_class_str, + attributes, + params + ) + + +class ExecutorClient(base.Executor): + """RPC Executor client.""" + + def __init__(self, transport): + """Constructs an RPC client for the Executor. + + :param transport: Messaging transport. + :type transport: Transport. + """ + serializer = auth_ctx.RpcContextSerializer( + auth_ctx.JsonPayloadSerializer() + ) + + self.topic = cfg.CONF.executor.topic + self._client = messaging.RPCClient( + transport, + messaging.Target(), + serializer=serializer + ) + + def run_action(self, action_ex_id, action_class_str, attributes, + action_params, target=None): + """Sends a request to run action to executor.""" + + kwargs = { + 'action_ex_id': action_ex_id, + 'action_class_str': action_class_str, + 'attributes': attributes, + 'params': action_params + } + + self._client.prepare(topic=self.topic, server=target).cast( + auth_ctx.ctx(), + 'run_action', + **kwargs + )