diff --git a/.gitignore b/.gitignore deleted file mode 100644 index d600e20..0000000 --- a/.gitignore +++ /dev/null @@ -1,21 +0,0 @@ -*.pyc -*.dat -TAGS -*.egg-info -build -.coverage -.tox -cover -venv -.venv -*.sublime-workspace -*.sqlite -var/* -etc/*.conf -etc/*.ini -AUTHORS -ChangeLog -dist -bufunfa/versioninfo -doc/source/api/* -doc/build/* diff --git a/.gitreview b/.gitreview deleted file mode 100644 index f8e492e..0000000 --- a/.gitreview +++ /dev/null @@ -1,4 +0,0 @@ -[gerrit] -host=review.openstack.org -port=29418 -project=stackforge/bufunfa.git diff --git a/HACKING.rst b/HACKING.rst deleted file mode 100644 index ec1a713..0000000 --- a/HACKING.rst +++ /dev/null @@ -1,201 +0,0 @@ -Glance Style Commandments -======================= - -- Step 1: Read http://www.python.org/dev/peps/pep-0008/ -- Step 2: Read http://www.python.org/dev/peps/pep-0008/ again -- Step 3: Read on - - -General -------- -- Put two newlines between top-level code (funcs, classes, etc) -- Put one newline between methods in classes and anywhere else -- Do not write "except:", use "except Exception:" at the very least -- Include your name with TODOs as in "#TODO(termie)" -- Do not name anything the same name as a built-in or reserved word -- Do declare variable names that conflict with the debugger, such as the letter 'c' - -Imports -------- -- Do not make relative imports -- Order your imports by the full module path -- Organize your imports according to the following template - -Example:: - - # vim: tabstop=4 shiftwidth=4 softtabstop=4 - {{stdlib imports in human alphabetical order}} - \n - {{third-party lib imports in human alphabetical order}} - \n - {{glance imports in human alphabetical order}} - \n - \n - {{begin your code}} - - -Human Alphabetical Order Examples ---------------------------------- -Example:: - - import httplib - import logging - import random - import StringIO - import time - import unittest - - import eventlet - import webob.exc - - import glance.api.middleware - from glance.api import images - from glance.auth import users - import glance.common - from glance.endpoint import cloud - from glance import test - - -Docstrings ----------- - -Docstrings are required for all functions and methods. - -Docstrings should ONLY use triple-double-quotes (``"""``) - -Single-line docstrings should NEVER have extraneous whitespace -between enclosing triple-double-quotes. - -**INCORRECT** :: - - """ There is some whitespace between the enclosing quotes :( """ - -**CORRECT** :: - - """There is no whitespace between the enclosing quotes :)""" - -Docstrings that span more than one line should look like this: - -Example:: - - """ - Start the docstring on the line following the opening triple-double-quote - - If you are going to describe parameters and return values, use Sphinx, the - appropriate syntax is as follows. - - :param foo: the foo parameter - :param bar: the bar parameter - :returns: return_type -- description of the return value - :returns: description of the return value - :raises: AttributeError, KeyError - """ - -**DO NOT** leave an extra newline before the closing triple-double-quote. - - -Dictionaries/Lists ------------------- -If a dictionary (dict) or list object is longer than 80 characters, its items -should be split with newlines. Embedded iterables should have their items -indented. Additionally, the last item in the dictionary should have a trailing -comma. This increases readability and simplifies future diffs. - -Example:: - - my_dictionary = { - "image": { - "name": "Just a Snapshot", - "size": 2749573, - "properties": { - "user_id": 12, - "arch": "x86_64", - }, - "things": [ - "thing_one", - "thing_two", - ], - "status": "ACTIVE", - }, - } - - -Calling Methods ---------------- -Calls to methods 80 characters or longer should format each argument with -newlines. This is not a requirement, but a guideline:: - - unnecessarily_long_function_name('string one', - 'string two', - kwarg1=constants.ACTIVE, - kwarg2=['a', 'b', 'c']) - - -Rather than constructing parameters inline, it is better to break things up:: - - list_of_strings = [ - 'what_a_long_string', - 'not as long', - ] - - dict_of_numbers = { - 'one': 1, - 'two': 2, - 'twenty four': 24, - } - - object_one.call_a_method('string three', - 'string four', - kwarg1=list_of_strings, - kwarg2=dict_of_numbers) - - -Internationalization (i18n) Strings ------------------------------------ -In order to support multiple languages, we have a mechanism to support -automatic translations of exception and log strings. - -Example:: - - msg = _("An error occurred") - raise HTTPBadRequest(explanation=msg) - -If you have a variable to place within the string, first internationalize the -template string then do the replacement. - -Example:: - - msg = _("Missing parameter: %s") % ("flavor",) - LOG.error(msg) - -If you have multiple variables to place in the string, use keyword parameters. -This helps our translators reorder parameters when needed. - -Example:: - - msg = _("The server with id %(s_id)s has no key %(m_key)s") - LOG.error(msg % {"s_id": "1234", "m_key": "imageId"}) - - -Creating Unit Tests -------------------- -For every new feature, unit tests should be created that both test and -(implicitly) document the usage of said feature. If submitting a patch for a -bug that had no unit test, a new passing unit test should be added. If a -submitted bug fix does have a unit test, be sure to add a new one that fails -without the patch and passes with the patch. - - -openstack-common ----------------- - -A number of modules from openstack-common are imported into the project. - -These modules are "incubating" in openstack-common and are kept in sync -with the help of openstack-common's update.py script. See: - - http://wiki.openstack.org/CommonLibrary#Incubation - -The copy of the code should never be directly modified here. Please -always update openstack-common first and then run the script to copy -the changes across. \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d645695..0000000 --- a/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index e5cc71d..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,17 +0,0 @@ -include AUTHORS -include ChangeLog -include HACKING.rst -include LICENSE -include README.rst -include MANIFEST.in pylintrc -include openstack-common.conf -include babel.cfg -graft templates -include heat/cloudinit/config -include heat/cloudinit/loguserdata.sh -include heat/cloudinit/part-handler.py -include heat/db/sqlalchemy/migrate_repo/migrate.cfg -graft etc -graft docs -graft tools - diff --git a/README.rst b/README.rst index f0f79a1..9006052 100644 --- a/README.rst +++ b/README.rst @@ -1,22 +1,7 @@ -Bufunfa +This project is no longer maintained. -'Bufunfa' Brazilian / Portugese short for 'Money' is a ChargeBack / Billing -software in OpenStack / Cloud style. +The contents of this repository are still available in the Git source code +management system. To see the contents of this repository before it reached +its end of life, please check out the previous commit with +"git checkout HEAD^1". -Overview: - Central: - The main component that's interacted with using MQ RPC - - typically Rabbit - Recorder: - The recorder typically fetches data from a external source like - Ceilometer and transforms it into something useful to the system. - It calls a procedure via the MQ RPC to store a "Record" which is the - thing described above. - - A Recorder houses multiple plugins loaded via 'stevedore' - - API: - Standard WSGI web services REST API. - - -Docs: http://bufunfa.rtfd.org diff --git a/bin/bufunfa-api b/bin/bufunfa-api deleted file mode 100644 index 8788082..0000000 --- a/bin/bufunfa-api +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import os -import sys -import eventlet -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common import service -from bufunfa.api import service as api_service - -eventlet.monkey_patch() - -config_files = cfg.find_config_files(project='bufunfa', - prog='bufunfa-api') - -if os.path.exists('./etc/bufunfa-api.conf'): - config_files.append('./etc/bufunfa-api.conf') - -cfg.CONF(sys.argv[1:], project='bufunfa', prog='bufunfa-api', - default_config_files=config_files) - -logging.setup('bufunfa') - -launcher = service.launch(api_service.Service()) -launcher.wait() diff --git a/bin/bufunfa-central b/bin/bufunfa-central deleted file mode 100644 index 03951d8..0000000 --- a/bin/bufunfa-central +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import eventlet -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log -from bufunfa.openstack.common import service -from bufunfa.central.service import Service - -eventlet.monkey_patch() - -config_files = cfg.find_config_files(project="bufunfa", prog="bufunfa-central") - -if os.path.exists('./etc/bufunfa-central.conf'): - config_files.append('./etc/bufunfa-central.conf') - -cfg.CONF(sys.argv[1:], project="bufunfa", prog="bufunfa-central", - default_config_files=config_files) - -log.setup("bufunfa") - -launcher = service.launch(Service()) -launcher.wait() diff --git a/bin/bufunfa-recorder b/bin/bufunfa-recorder deleted file mode 100644 index 027adc7..0000000 --- a/bin/bufunfa-recorder +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import os -import sys -import eventlet - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log -from bufunfa.openstack.common import service -from bufunfa.recorder.service import RecordService - -eventlet.monkey_patch() - -config_files = cfg.find_config_files(project="bufunfa", - prog="bufunfa-recorder-sync") - -if os.path.exists('./etc/bufunfa-recorder.conf'): - config_files.append('./etc/bufunfa-recorder.conf') - -cfg.CONF(sys.argv[1:], project="bufunfa", prog="bufunfa-recorder-sync", - default_config_files=config_files) - -log.setup('bufunfa') - -launcher = service.launch(RecordService()) -launcher.wait() diff --git a/bin/bufunfa-sync b/bin/bufunfa-sync deleted file mode 100644 index 42f47f0..0000000 --- a/bin/bufunfa-sync +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/env python - -import sys -from bufunfa import storage -from bufunfa.openstack.common import cfg - -if __name__ == '__main__': - cfg.CONF(sys.argv[1:]) - - storage.register_opts(cfg.CONF) - conn = storage.get_connection(cfg.CONF) - - conn.setup_schema() diff --git a/bufunfa/__init__.py b/bufunfa/__init__.py deleted file mode 100644 index ccd89c3..0000000 --- a/bufunfa/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License.:q - -import os -import socket -from bufunfa.openstack.common import cfg - - -cfg.CONF.register_opts([ - cfg.StrOpt('pybasedir', - default=os.path.abspath(os.path.join(os.path.dirname(__file__), - '../')), - help='Directory where the nova python module is installed'), - cfg.StrOpt('host', default=socket.gethostname(), - help='Name of this node'), - cfg.StrOpt('control-exchange', default='bufunfa', - help='AMQP exchange to connect to if using RabbitMQ or Qpid'), - cfg.StrOpt('central-topic', - default='bufunfa.central', - help='Central Topic'), - cfg.StrOpt('worker-topic', default='bufunfa.recorder', - help='Worker Topic'), - cfg.StrOpt('state-path', default='$pybasedir', help='State Path') -]) diff --git a/bufunfa/api/__init__.py b/bufunfa/api/__init__.py deleted file mode 100644 index f1adb4b..0000000 --- a/bufunfa/api/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied from Moniker -import flask -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import jsonutils - - -# Replace the json module used by flask with the one from -# bufunfa.openstack.common so we can take advantage of the fact that it knows -# how to serialize more complex objects. -flask.helpers.json = jsonutils - - -cfg.CONF.register_opts([ - cfg.StrOpt('api_host', default='0.0.0.0', - help='API Host'), - cfg.IntOpt('api_port', default=9001, - help='API Port Number'), - cfg.StrOpt('api_paste_config', default='bufunfa-api-paste.ini', - help='File name for the paste.deploy config for bufunfa-api'), - cfg.StrOpt('auth_strategy', default='noauth', - help='The strategy to use for auth. Supports noauth or ' - 'keystone'), -]) diff --git a/bufunfa/api/auth.py b/bufunfa/api/auth.py deleted file mode 100644 index 22111dd..0000000 --- a/bufunfa/api/auth.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied from Moniker -from bufunfa.openstack.common.context import RequestContext -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log as logging -from bufunfa import wsgi - -LOG = logging.getLogger(__name__) - - -def pipeline_factory(loader, global_conf, **local_conf): - """ - A paste pipeline replica that keys off of auth_strategy. - - Code nabbed from cinder. - """ - pipeline = local_conf[cfg.CONF.auth_strategy] - pipeline = pipeline.split() - filters = [loader.get_filter(n) for n in pipeline[:-1]] - app = loader.get_app(pipeline[-1]) - filters.reverse() - for filter in filters: - app = filter(app) - return app - - -class KeystoneContextMiddleware(wsgi.Middleware): - def process_request(self, request): - headers = request.headers - context = RequestContext(auth_tok=headers.get('X-Auth-Token'), - user=headers.get('X-User-ID'), - tenant=headers.get('X-Tenant-ID')) - request.environ['context'] = context - - -class NoAuthMiddleware(wsgi.Middleware): - def process_request(self, request): - request.environ['context'] = RequestContext() diff --git a/bufunfa/api/service.py b/bufunfa/api/service.py deleted file mode 100644 index fab2974..0000000 --- a/bufunfa/api/service.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied from Moniker -from paste import deploy -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common import wsgi -from bufunfa.openstack.common import cfg -from bufunfa import utils - - -LOG = logging.getLogger(__name__) - - -class Service(wsgi.Service): - def __init__(self, backlog=128, threads=1000): - super(Service, self).__init__(threads) - - self.host = cfg.CONF.api_host - self.port = cfg.CONF.api_port - self.backlog = backlog - - config_path = utils.find_config(cfg.CONF.api_paste_config) - - self.application = deploy.loadapp("config:%s" % config_path, - name='osapi_billing') - - def start(self): - return super(Service, self).start(application=self.application, - port=self.port, host=self.host, - backlog=self.backlog) diff --git a/bufunfa/api/v1.py b/bufunfa/api/v1.py deleted file mode 100644 index d686172..0000000 --- a/bufunfa/api/v1.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import flask - -from bufunfa.openstack.common import log -from bufunfa.central import api as central_api - -LOG = log.getLogger(__name__) - -blueprint = flask.Blueprint('v1', __name__) - - -@blueprint.route('/rates', methods=['POST']) -def add_rate(): - context = flask.request.environ.get('context') - values = flask.request.json - - rate = central_api.add_rate(context, values) - return flask.jsonify(rate=rate) - - -@blueprint.route('/rates', methods=['GET']) -def get_rates(): - context = flask.request.environ.get('context') - rates = central_api.get_rates(context) - return flask.jsonify(rates=rates) - - -@blueprint.route('/rates/', methods=['PUT']) -def update_rate(rate_id): - context = flask.request.environ.get('context') - values = flask.request.json - - rate = central_api.update_rate(context, rate_id, values) - return flask.jsonify(rate) - - -@blueprint.route('/rates/', methods=['DELETE']) -def delete_rate(rate_id): - context = flask.request.environ.get('context') - central_api.delete_rate(context, rate_id) - - -@blueprint.route('/accounts', methods=['POST']) -def add_account(): - context = flask.request.environ.get('context') - values = flask.request.json - - account = central_api.add_account(context, values) - return flask.jsonify(account=account) - - -@blueprint.route('/accounts', methods=['GET']) -def get_accounts(): - context = flask.request.environ.get('context') - accounts = central_api.get_accounts(context) - return flask.jsonify(accounts=accounts) - - -@blueprint.route('/accounts/', methods=['PUT']) -def update_account(account_id): - context = flask.request.environ.get('context') - values = flask.request.json - - account = central_api.update_account(context, account_id, values) - return flask.jsonify(account=account) - - -@blueprint.route('/accounts/', methods=['DELETE']) -def delete_account(account_id): - context = flask.request.environ.get('context') - central_api.delete_account(context, account_id) - - -@blueprint.route('/system_accounts', methods=['POST']) -def add_system_account(): - context = flask.request.environ.get('context') - values = flask.request.json - - account = central_api.add_system_account(context, values) - return flask.jsonify(system_account=account) - - -@blueprint.route('/system_accounts', methods=['GET']) -def get_system_accounts(): - context = flask.request.environ.get('context') - accounts = central_api.get_system_accounts(context) - return flask.jsonify(system_accounts=accounts) - - -@blueprint.route('/system_accounts/', methods=['PUT']) -def update_system_account(account_id): - context = flask.request.environ.get('context') - values = flask.request.json - - account = central_api.update_system_account(context, account_id, values) - return flask.jsonify(system_account=account) - - -@blueprint.route('/system_accounts/', methods=['DELETE']) -def delete_system_account(account_id): - context = flask.request.environ.get('context') - central_api.delete_account(context, account_id) - - -@blueprint.route('/record', methods=['POST']) -def process_record(): - context = flask.request.environ.get('context') - values = flask.request.json - - record = central_api.process_record(context, values) - return flask.jsonify(record) - - -def factory(global_config, **local_conf): - app = flask.Flask('bufunfa.api.v1') - app.register_blueprint(blueprint) - return app diff --git a/bufunfa/central/__init__.py b/bufunfa/central/__init__.py deleted file mode 100644 index 726e1f5..0000000 --- a/bufunfa/central/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/bufunfa/central/api.py b/bufunfa/central/api.py deleted file mode 100644 index 6f634fc..0000000 --- a/bufunfa/central/api.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied verbatim from Moniker -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common.rpc.proxy import RpcProxy - -DEFAULT_VERSION = "1.0" - -LOG = logging.getLogger(__name__) -RPC = RpcProxy(cfg.CONF.central_topic, DEFAULT_VERSION) - - -def add_rate(context, values): - msg = { - "method": "add_rate", - "args": { - "values": values - } - } - return RPC.call(context, msg) - - -def get_rate(context, rate_id): - msg = { - "method": "get_rate", - "args": { - "rate_id": rate_id - } - } - return RPC.call(context, msg) - - -def get_rates(context): - msg = { - "method": "get_rates", - } - return RPC.call(context, msg) - - -def update_rate(context, rate_id, values): - msg = { - "method": "update_rate", - "args": { - "rate_id": rate_id, - "values": values - } - } - return RPC.call(context, msg) - - -def delete_rate(context, rate_id): - msg = { - "method": "delete_rate", - "args": { - "rate_id": rate_id - } - } - return RPC.call(context, msg) - - -def add_account(context, values): - msg = { - "method": "add_account", - "args": { - "values": values - } - } - return RPC.call(context, msg) - - -def get_account(context, account_id): - msg = { - "method": "get_account", - "args": { - "account_id": account_id - } - } - return RPC.call(context, msg) - - -def get_accounts(context): - msg = { - "method": "get_accounts", - } - return RPC.call(context, msg) - - -def update_account(context, account_id, values): - msg = { - "method": "update_account", - "args": { - "account_id": account_id, - "values": values - } - } - return RPC.call(context, msg) - - -def delete_account(context, account_id): - msg = { - "method": "delete_account", - "args": { - "account_id": account_id - } - } - return RPC.call(context, msg) - - -def add_system_account(context, values): - msg = { - "method": "add_system_account", - "args": { - "values": values - } - } - return RPC.call(context, msg) - - -def get_system_account(context, account_id): - msg = { - "method": "get_system_account", - "args": { - "account_id": account_id - } - } - return RPC.call(context, msg) - - -def get_system_accounts(context): - msg = { - "method": "get_system_accounts", - } - return RPC.call(context, msg) - - -def update_system_account(context, account_id, values): - msg = { - "method": "update_system_account", - "args": { - "account_id": account_id, - "values": values - } - } - return RPC.call(context, msg) - - -def delete_system_account(context, account_id): - msg = { - "method": "delete_system_account", - "args": { - "account_id": account_id - } - } - return RPC.call(context, msg) - - -def set_polled_at(context, account_id, time): - msg = { - "method": "set_polled_at", - "args": { - "account_id": account_id, - "time": time - } - } - return RPC.call(context, msg) - - -def process_record(context, values): - msg = { - "method": "process_record", - "args": { - "values": values - } - } - return RPC.call(context, msg) - - -def process_records(context, records): - msg = { - 'method': 'process_records', - 'args': { - 'records': records - } - } - return RPC.call(context, msg) diff --git a/bufunfa/central/service.py b/bufunfa/central/service.py deleted file mode 100644 index fd1e26b..0000000 --- a/bufunfa/central/service.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied verbatim from Moniker -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log -from bufunfa.openstack.common import timeutils -from bufunfa.openstack.common.rpc import service as rpc_service -from bufunfa import exceptions -from bufunfa import storage - - -LOG = log.getLogger(__name__) - - -class Service(rpc_service.Service): - def __init__(self, *args, **kw): - kw.update( - host=cfg.CONF.host, - topic=cfg.CONF.central_topic) - - super(Service, self).__init__(*args, **kw) - - self.storage_conn = storage.get_connection(cfg.CONF) - - def add_rate(self, context, values): - return self.storage_conn.add_rate(context, values) - - def get_rate(self, context, rate_id): - return self.storage_conn.get_rate(context, rate_id) - - def get_rates(self, context): - return self.storage_conn.get_rates(context) - - def update_rate(self, context, rate_id, values): - return self.storage_conn.update_rate(context, rate_id, values) - - def delete_rate(self, context, rate_id): - return self.storage_conn.delete_rate(context, rate_id) - - def add_account(self, context, values): - return self.storage_conn.add_account(context, values) - - def get_account(self, context, account_id): - return self.storage_conn.get_account(context, account_id) - - def get_accounts(self, context): - return self.storage_conn.get_accounts(context) - - def update_account(self, context, account_id, values): - return self.storage_conn.update_account(context, account_id, values) - - def delete_account(self, context, account_id): - return self.storage_conn.delete_rate(context, account_id) - - def add_system_account(self, context, values): - return self.storage_conn.add_system_account(context, values) - - def get_system_account(self, context, account_id): - return self.storage_conn.get_system_account(context, account_id) - - def get_system_accounts(self, context): - return self.storage_conn.get_system_accounts(context) - - def update_system_account(self, context, account_id, values): - return self.storage_conn.update_account(context, account_id, values) - - def delete_system_account(self, context, account_id): - return self.storage_conn.delete_rate(context, account_id) - - def set_polled_at(self, context, account_id, time): - """ - Set when the account was last polled in the system - - :param context: RPC context - :param account_id: The Account ID in the System - :param time_stamp: Timestamp of when it was polled - """ - time = timeutils.parse_strtime(time) - - account = self.storage_conn.get_system_account(context, account_id) - polled_at = account['polled_at'] - - if polled_at and time < polled_at: - raise exceptions.TooOld("Timestamp is older then the last poll") - - return self.storage_conn.update_system_account(context, account_id, - {'polled_at': time}) - - def process_records(self, context, records): - """ - Process records in a batch - - :param context: RPC context - :param records: A list of records - """ - for record in records: - self.process_record(context, record) - - def process_record(self, context, values): - """ - Process a Record - - :param context: RPC context - :param values: Values for the record - """ - # NOTE: Add the system if it doesn't exist.. - try: - self.storage_conn.get_system_account( - context, values['account_id']) - except exceptions.NotFound: - self.storage_conn.add_system_account( - context, - {'id': values['account_id']}) - - self.storage_conn.add_record(context, values) diff --git a/bufunfa/exceptions.py b/bufunfa/exceptions.py deleted file mode 100644 index dcd1d07..0000000 --- a/bufunfa/exceptions.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class Base(Exception): - pass - - -class ConfigNotFound(Base): - pass - - -class InvalidObject(Base): - pass - - -class Forbidden(Base): - pass - - -class InvalidSortKey(Base): - pass - - -class Duplicate(Base): - pass - - -class NotFound(Base): - pass - - -class TooOld(Base): - pass diff --git a/bufunfa/openstack/__init__.py b/bufunfa/openstack/__init__.py deleted file mode 100644 index 9014e56..0000000 --- a/bufunfa/openstack/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This ensures the bufunfa.openstack namespace is defined -try: - import pkg_resources - pkg_resources.declare_namespace(__name__) -except ImportError: - import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/bufunfa/openstack/common/__init__.py b/bufunfa/openstack/common/__init__.py deleted file mode 100644 index afcfd7e..0000000 --- a/bufunfa/openstack/common/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# TODO(jaypipes) Code in this module is intended to be ported to the eventual -# bufunfa.openstack-common library diff --git a/bufunfa/openstack/common/cfg.py b/bufunfa/openstack/common/cfg.py deleted file mode 100644 index bd3f580..0000000 --- a/bufunfa/openstack/common/cfg.py +++ /dev/null @@ -1,1653 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -r""" -Configuration options which may be set on the command line or in config files. - -The schema for each option is defined using the Opt sub-classes, e.g.: - -:: - - common_opts = [ - cfg.StrOpt('bind_host', - default='0.0.0.0', - help='IP address to listen on'), - cfg.IntOpt('bind_port', - default=9292, - help='Port number to listen on') - ] - -Options can be strings, integers, floats, booleans, lists or 'multi strings':: - - enabled_apis_opt = cfg.ListOpt('enabled_apis', - default=['ec2', 'osapi_compute'], - help='List of APIs to enable by default') - - DEFAULT_EXTENSIONS = [ - 'nova.api.openstack.compute.contrib.standard_extensions' - ] - osapi_compute_extension_opt = cfg.MultiStrOpt('osapi_compute_extension', - default=DEFAULT_EXTENSIONS) - -Option schemas are registered with the config manager at runtime, but before -the option is referenced:: - - class ExtensionManager(object): - - enabled_apis_opt = cfg.ListOpt(...) - - def __init__(self, conf): - self.conf = conf - self.conf.register_opt(enabled_apis_opt) - ... - - def _load_extensions(self): - for ext_factory in self.conf.osapi_compute_extension: - .... - -A common usage pattern is for each option schema to be defined in the module or -class which uses the option:: - - opts = ... - - def add_common_opts(conf): - conf.register_opts(opts) - - def get_bind_host(conf): - return conf.bind_host - - def get_bind_port(conf): - return conf.bind_port - -An option may optionally be made available via the command line. Such options -must registered with the config manager before the command line is parsed (for -the purposes of --help and CLI arg validation):: - - cli_opts = [ - cfg.BoolOpt('verbose', - short='v', - default=False, - help='Print more verbose output'), - cfg.BoolOpt('debug', - short='d', - default=False, - help='Print debugging output'), - ] - - def add_common_opts(conf): - conf.register_cli_opts(cli_opts) - -The config manager has two CLI options defined by default, --config-file -and --config-dir:: - - class ConfigOpts(object): - - def __call__(self, ...): - - opts = [ - MultiStrOpt('config-file', - ...), - StrOpt('config-dir', - ...), - ] - - self.register_cli_opts(opts) - -Option values are parsed from any supplied config files using -openstack.common.iniparser. If none are specified, a default set is used -e.g. glance-api.conf and glance-common.conf:: - - glance-api.conf: - [DEFAULT] - bind_port = 9292 - - glance-common.conf: - [DEFAULT] - bind_host = 0.0.0.0 - -Option values in config files override those on the command line. Config files -are parsed in order, with values in later files overriding those in earlier -files. - -The parsing of CLI args and config files is initiated by invoking the config -manager e.g.:: - - conf = ConfigOpts() - conf.register_opt(BoolOpt('verbose', ...)) - conf(sys.argv[1:]) - if conf.verbose: - ... - -Options can be registered as belonging to a group:: - - rabbit_group = cfg.OptGroup(name='rabbit', - title='RabbitMQ options') - - rabbit_host_opt = cfg.StrOpt('host', - default='localhost', - help='IP/hostname to listen on'), - rabbit_port_opt = cfg.IntOpt('port', - default=5672, - help='Port number to listen on') - - def register_rabbit_opts(conf): - conf.register_group(rabbit_group) - # options can be registered under a group in either of these ways: - conf.register_opt(rabbit_host_opt, group=rabbit_group) - conf.register_opt(rabbit_port_opt, group='rabbit') - -If it no group attributes are required other than the group name, the group -need not be explicitly registered e.g. - - def register_rabbit_opts(conf): - # The group will automatically be created, equivalent calling:: - # conf.register_group(OptGroup(name='rabbit')) - conf.register_opt(rabbit_port_opt, group='rabbit') - -If no group is specified, options belong to the 'DEFAULT' section of config -files:: - - glance-api.conf: - [DEFAULT] - bind_port = 9292 - ... - - [rabbit] - host = localhost - port = 5672 - use_ssl = False - userid = guest - password = guest - virtual_host = / - -Command-line options in a group are automatically prefixed with the -group name:: - - --rabbit-host localhost --rabbit-port 9999 - -Option values in the default group are referenced as attributes/properties on -the config manager; groups are also attributes on the config manager, with -attributes for each of the options associated with the group:: - - server.start(app, conf.bind_port, conf.bind_host, conf) - - self.connection = kombu.connection.BrokerConnection( - hostname=conf.rabbit.host, - port=conf.rabbit.port, - ...) - -Option values may reference other values using PEP 292 string substitution:: - - opts = [ - cfg.StrOpt('state_path', - default=os.path.join(os.path.dirname(__file__), '../'), - help='Top-level directory for maintaining nova state'), - cfg.StrOpt('sqlite_db', - default='nova.sqlite', - help='file name for sqlite'), - cfg.StrOpt('sql_connection', - default='sqlite:///$state_path/$sqlite_db', - help='connection string for sql database'), - ] - -Note that interpolation can be avoided by using '$$'. - -For command line utilities that dispatch to other command line utilities, the -disable_interspersed_args() method is available. If this this method is called, -then parsing e.g.:: - - script --verbose cmd --debug /tmp/mything - -will no longer return:: - - ['cmd', '/tmp/mything'] - -as the leftover arguments, but will instead return:: - - ['cmd', '--debug', '/tmp/mything'] - -i.e. argument parsing is stopped at the first non-option argument. - -Options may be declared as required so that an error is raised if the user -does not supply a value for the option. - -Options may be declared as secret so that their values are not leaked into -log files: - - opts = [ - cfg.StrOpt('s3_store_access_key', secret=True), - cfg.StrOpt('s3_store_secret_key', secret=True), - ... - ] - -This module also contains a global instance of the CommonConfigOpts class -in order to support a common usage pattern in OpenStack: - - from openstack.common import cfg - - opts = [ - cfg.StrOpt('bind_host', default='0.0.0.0'), - cfg.IntOpt('bind_port', default=9292), - ] - - CONF = cfg.CONF - CONF.register_opts(opts) - - def start(server, app): - server.start(app, CONF.bind_port, CONF.bind_host) - -""" - -import collections -import copy -import functools -import glob -import optparse -import os -import string -import sys - -from bufunfa.openstack.common import iniparser - - -class Error(Exception): - """Base class for cfg exceptions.""" - - def __init__(self, msg=None): - self.msg = msg - - def __str__(self): - return self.msg - - -class ArgsAlreadyParsedError(Error): - """Raised if a CLI opt is registered after parsing.""" - - def __str__(self): - ret = "arguments already parsed" - if self.msg: - ret += ": " + self.msg - return ret - - -class NoSuchOptError(Error, AttributeError): - """Raised if an opt which doesn't exist is referenced.""" - - def __init__(self, opt_name, group=None): - self.opt_name = opt_name - self.group = group - - def __str__(self): - if self.group is None: - return "no such option: %s" % self.opt_name - else: - return "no such option in group %s: %s" % (self.group.name, - self.opt_name) - - -class NoSuchGroupError(Error): - """Raised if a group which doesn't exist is referenced.""" - - def __init__(self, group_name): - self.group_name = group_name - - def __str__(self): - return "no such group: %s" % self.group_name - - -class DuplicateOptError(Error): - """Raised if multiple opts with the same name are registered.""" - - def __init__(self, opt_name): - self.opt_name = opt_name - - def __str__(self): - return "duplicate option: %s" % self.opt_name - - -class RequiredOptError(Error): - """Raised if an option is required but no value is supplied by the user.""" - - def __init__(self, opt_name, group=None): - self.opt_name = opt_name - self.group = group - - def __str__(self): - if self.group is None: - return "value required for option: %s" % self.opt_name - else: - return "value required for option: %s.%s" % (self.group.name, - self.opt_name) - - -class TemplateSubstitutionError(Error): - """Raised if an error occurs substituting a variable in an opt value.""" - - def __str__(self): - return "template substitution error: %s" % self.msg - - -class ConfigFilesNotFoundError(Error): - """Raised if one or more config files are not found.""" - - def __init__(self, config_files): - self.config_files = config_files - - def __str__(self): - return ('Failed to read some config files: %s' % - string.join(self.config_files, ',')) - - -class ConfigFileParseError(Error): - """Raised if there is an error parsing a config file.""" - - def __init__(self, config_file, msg): - self.config_file = config_file - self.msg = msg - - def __str__(self): - return 'Failed to parse %s: %s' % (self.config_file, self.msg) - - -class ConfigFileValueError(Error): - """Raised if a config file value does not match its opt type.""" - pass - - -def _fixpath(p): - """Apply tilde expansion and absolutization to a path.""" - return os.path.abspath(os.path.expanduser(p)) - - -def _get_config_dirs(project=None): - """Return a list of directors where config files may be located. - - :param project: an optional project name - - If a project is specified, following directories are returned:: - - ~/.${project}/ - ~/ - /etc/${project}/ - /etc/ - - Otherwise, these directories:: - - ~/ - /etc/ - """ - cfg_dirs = [ - _fixpath(os.path.join('~', '.' + project)) if project else None, - _fixpath('~'), - os.path.join('/etc', project) if project else None, - '/etc' - ] - - return filter(bool, cfg_dirs) - - -def _search_dirs(dirs, basename, extension=""): - """Search a list of directories for a given filename. - - Iterator over the supplied directories, returning the first file - found with the supplied name and extension. - - :param dirs: a list of directories - :param basename: the filename, e.g. 'glance-api' - :param extension: the file extension, e.g. '.conf' - :returns: the path to a matching file, or None - """ - for d in dirs: - path = os.path.join(d, '%s%s' % (basename, extension)) - if os.path.exists(path): - return path - - -def find_config_files(project=None, prog=None, extension='.conf'): - """Return a list of default configuration files. - - :param project: an optional project name - :param prog: the program name, defaulting to the basename of sys.argv[0] - :param extension: the type of the config file - - We default to two config files: [${project}.conf, ${prog}.conf] - - And we look for those config files in the following directories:: - - ~/.${project}/ - ~/ - /etc/${project}/ - /etc/ - - We return an absolute path for (at most) one of each the default config - files, for the topmost directory it exists in. - - For example, if project=foo, prog=bar and /etc/foo/foo.conf, /etc/bar.conf - and ~/.foo/bar.conf all exist, then we return ['/etc/foo/foo.conf', - '~/.foo/bar.conf'] - - If no project name is supplied, we only look for ${prog.conf}. - """ - if prog is None: - prog = os.path.basename(sys.argv[0]) - - cfg_dirs = _get_config_dirs(project) - - config_files = [] - if project: - config_files.append(_search_dirs(cfg_dirs, project, extension)) - config_files.append(_search_dirs(cfg_dirs, prog, extension)) - - return filter(bool, config_files) - - -def _is_opt_registered(opts, opt): - """Check whether an opt with the same name is already registered. - - The same opt may be registered multiple times, with only the first - registration having any effect. However, it is an error to attempt - to register a different opt with the same name. - - :param opts: the set of opts already registered - :param opt: the opt to be registered - :returns: True if the opt was previously registered, False otherwise - :raises: DuplicateOptError if a naming conflict is detected - """ - if opt.dest in opts: - if opts[opt.dest]['opt'] != opt: - raise DuplicateOptError(opt.name) - return True - else: - return False - - -class Opt(object): - - """Base class for all configuration options. - - An Opt object has no public methods, but has a number of public string - properties: - - name: - the name of the option, which may include hyphens - dest: - the (hyphen-less) ConfigOpts property which contains the option value - short: - a single character CLI option name - default: - the default value of the option - metavar: - the name shown as the argument to a CLI option in --help output - help: - an string explaining how the options value is used - """ - multi = False - - def __init__(self, name, dest=None, short=None, default=None, - metavar=None, help=None, secret=False, required=False, - deprecated_name=None): - """Construct an Opt object. - - The only required parameter is the option's name. However, it is - common to also supply a default and help string for all options. - - :param name: the option's name - :param dest: the name of the corresponding ConfigOpts property - :param short: a single character CLI option name - :param default: the default value of the option - :param metavar: the option argument to show in --help - :param help: an explanation of how the option is used - :param secret: true iff the value should be obfuscated in log output - :param required: true iff a value must be supplied for this option - :param deprecated_name: deprecated name option. Acts like an alias - """ - self.name = name - if dest is None: - self.dest = self.name.replace('-', '_') - else: - self.dest = dest - self.short = short - self.default = default - self.metavar = metavar - self.help = help - self.secret = secret - self.required = required - if deprecated_name is not None: - self.deprecated_name = deprecated_name.replace('-', '_') - else: - self.deprecated_name = None - - def __ne__(self, another): - return vars(self) != vars(another) - - def _get_from_config_parser(self, cparser, section): - """Retrieves the option value from a MultiConfigParser object. - - This is the method ConfigOpts uses to look up the option value from - config files. Most opt types override this method in order to perform - type appropriate conversion of the returned value. - - :param cparser: a ConfigParser object - :param section: a section name - """ - return self._cparser_get_with_deprecated(cparser, section) - - def _cparser_get_with_deprecated(self, cparser, section): - """If cannot find option as dest try deprecated_name alias.""" - if self.deprecated_name is not None: - return cparser.get(section, [self.dest, self.deprecated_name]) - return cparser.get(section, [self.dest]) - - def _add_to_cli(self, parser, group=None): - """Makes the option available in the command line interface. - - This is the method ConfigOpts uses to add the opt to the CLI interface - as appropriate for the opt type. Some opt types may extend this method, - others may just extend the helper methods it uses. - - :param parser: the CLI option parser - :param group: an optional OptGroup object - """ - container = self._get_optparse_container(parser, group) - kwargs = self._get_optparse_kwargs(group) - prefix = self._get_optparse_prefix('', group) - self._add_to_optparse(container, self.name, self.short, kwargs, prefix, - self.deprecated_name) - - def _add_to_optparse(self, container, name, short, kwargs, prefix='', - deprecated_name=None): - """Add an option to an optparse parser or group. - - :param container: an optparse.OptionContainer object - :param name: the opt name - :param short: the short opt name - :param kwargs: the keyword arguments for add_option() - :param prefix: an optional prefix to prepend to the opt name - :raises: DuplicateOptError if a naming confict is detected - """ - args = ['--' + prefix + name] - if short: - args += ['-' + short] - if deprecated_name: - args += ['--' + prefix + deprecated_name] - for a in args: - if container.has_option(a): - raise DuplicateOptError(a) - container.add_option(*args, **kwargs) - - def _get_optparse_container(self, parser, group): - """Returns an optparse.OptionContainer. - - :param parser: an optparse.OptionParser - :param group: an (optional) OptGroup object - :returns: an optparse.OptionGroup if a group is given, else the parser - """ - if group is not None: - return group._get_optparse_group(parser) - else: - return parser - - def _get_optparse_kwargs(self, group, **kwargs): - """Build a dict of keyword arguments for optparse's add_option(). - - Most opt types extend this method to customize the behaviour of the - options added to optparse. - - :param group: an optional group - :param kwargs: optional keyword arguments to add to - :returns: a dict of keyword arguments - """ - dest = self.dest - if group is not None: - dest = group.name + '_' + dest - kwargs.update({'dest': dest, - 'metavar': self.metavar, - 'help': self.help, }) - return kwargs - - def _get_optparse_prefix(self, prefix, group): - """Build a prefix for the CLI option name, if required. - - CLI options in a group are prefixed with the group's name in order - to avoid conflicts between similarly named options in different - groups. - - :param prefix: an existing prefix to append to (e.g. 'no' or '') - :param group: an optional OptGroup object - :returns: a CLI option prefix including the group name, if appropriate - """ - if group is not None: - return group.name + '-' + prefix - else: - return prefix - - -class StrOpt(Opt): - """ - String opts do not have their values transformed and are returned as - str objects. - """ - pass - - -class BoolOpt(Opt): - - """ - Bool opts are set to True or False on the command line using --optname or - --noopttname respectively. - - In config files, boolean values are case insensitive and can be set using - 1/0, yes/no, true/false or on/off. - """ - - _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, - '0': False, 'no': False, 'false': False, 'off': False} - - def _get_from_config_parser(self, cparser, section): - """Retrieve the opt value as a boolean from ConfigParser.""" - def convert_bool(v): - value = self._boolean_states.get(v.lower()) - if value is None: - raise ValueError('Unexpected boolean value %r' % v) - - return value - - return [convert_bool(v) for v in - self._cparser_get_with_deprecated(cparser, section)] - - def _add_to_cli(self, parser, group=None): - """Extends the base class method to add the --nooptname option.""" - super(BoolOpt, self)._add_to_cli(parser, group) - self._add_inverse_to_optparse(parser, group) - - def _add_inverse_to_optparse(self, parser, group): - """Add the --nooptname option to the option parser.""" - container = self._get_optparse_container(parser, group) - kwargs = self._get_optparse_kwargs(group, action='store_false') - prefix = self._get_optparse_prefix('no', group) - kwargs["help"] = "The inverse of --" + self.name - self._add_to_optparse(container, self.name, None, kwargs, prefix, - self.deprecated_name) - - def _get_optparse_kwargs(self, group, action='store_true', **kwargs): - """Extends the base optparse keyword dict for boolean options.""" - return super(BoolOpt, - self)._get_optparse_kwargs(group, action=action, **kwargs) - - -class IntOpt(Opt): - - """Int opt values are converted to integers using the int() builtin.""" - - def _get_from_config_parser(self, cparser, section): - """Retrieve the opt value as a integer from ConfigParser.""" - return [int(v) for v in self._cparser_get_with_deprecated(cparser, - section)] - - def _get_optparse_kwargs(self, group, **kwargs): - """Extends the base optparse keyword dict for integer options.""" - return super(IntOpt, - self)._get_optparse_kwargs(group, type='int', **kwargs) - - -class FloatOpt(Opt): - - """Float opt values are converted to floats using the float() builtin.""" - - def _get_from_config_parser(self, cparser, section): - """Retrieve the opt value as a float from ConfigParser.""" - return [float(v) for v in - self._cparser_get_with_deprecated(cparser, section)] - - def _get_optparse_kwargs(self, group, **kwargs): - """Extends the base optparse keyword dict for float options.""" - return super(FloatOpt, - self)._get_optparse_kwargs(group, type='float', **kwargs) - - -class ListOpt(Opt): - - """ - List opt values are simple string values separated by commas. The opt value - is a list containing these strings. - """ - - def _get_from_config_parser(self, cparser, section): - """Retrieve the opt value as a list from ConfigParser.""" - return [v.split(',') for v in - self._cparser_get_with_deprecated(cparser, section)] - - def _get_optparse_kwargs(self, group, **kwargs): - """Extends the base optparse keyword dict for list options.""" - return super(ListOpt, - self)._get_optparse_kwargs(group, - type='string', - action='callback', - callback=self._parse_list, - **kwargs) - - def _parse_list(self, option, opt, value, parser): - """An optparse callback for parsing an option value into a list.""" - setattr(parser.values, self.dest, value.split(',')) - - -class MultiStrOpt(Opt): - - """ - Multistr opt values are string opts which may be specified multiple times. - The opt value is a list containing all the string values specified. - """ - multi = True - - def _get_optparse_kwargs(self, group, **kwargs): - """Extends the base optparse keyword dict for multi str options.""" - return super(MultiStrOpt, - self)._get_optparse_kwargs(group, action='append') - - def _cparser_get_with_deprecated(self, cparser, section): - """If cannot find option as dest try deprecated_name alias.""" - if self.deprecated_name is not None: - return cparser.get(section, [self.dest, self.deprecated_name], - multi=True) - return cparser.get(section, [self.dest], multi=True) - - -class OptGroup(object): - - """ - Represents a group of opts. - - CLI opts in the group are automatically prefixed with the group name. - - Each group corresponds to a section in config files. - - An OptGroup object has no public methods, but has a number of public string - properties: - - name: - the name of the group - title: - the group title as displayed in --help - help: - the group description as displayed in --help - """ - - def __init__(self, name, title=None, help=None): - """Constructs an OptGroup object. - - :param name: the group name - :param title: the group title for --help - :param help: the group description for --help - """ - self.name = name - if title is None: - self.title = "%s options" % title - else: - self.title = title - self.help = help - - self._opts = {} # dict of dicts of (opt:, override:, default:) - self._optparse_group = None - - def _register_opt(self, opt): - """Add an opt to this group. - - :param opt: an Opt object - :returns: False if previously registered, True otherwise - :raises: DuplicateOptError if a naming conflict is detected - """ - if _is_opt_registered(self._opts, opt): - return False - - self._opts[opt.dest] = {'opt': opt} - - return True - - def _unregister_opt(self, opt): - """Remove an opt from this group. - - :param opt: an Opt object - """ - if opt.dest in self._opts: - del self._opts[opt.dest] - - def _get_optparse_group(self, parser): - """Build an optparse.OptionGroup for this group.""" - if self._optparse_group is None: - self._optparse_group = optparse.OptionGroup(parser, self.title, - self.help) - return self._optparse_group - - def _clear(self): - """Clear this group's option parsing state.""" - self._optparse_group = None - - -class ParseError(iniparser.ParseError): - def __init__(self, msg, lineno, line, filename): - super(ParseError, self).__init__(msg, lineno, line) - self.filename = filename - - def __str__(self): - return 'at %s:%d, %s: %r' % (self.filename, self.lineno, - self.msg, self.line) - - -class ConfigParser(iniparser.BaseParser): - def __init__(self, filename, sections): - super(ConfigParser, self).__init__() - self.filename = filename - self.sections = sections - self.section = None - - def parse(self): - with open(self.filename) as f: - return super(ConfigParser, self).parse(f) - - def new_section(self, section): - self.section = section - self.sections.setdefault(self.section, {}) - - def assignment(self, key, value): - if not self.section: - raise self.error_no_section() - - self.sections[self.section].setdefault(key, []) - self.sections[self.section][key].append('\n'.join(value)) - - def parse_exc(self, msg, lineno, line=None): - return ParseError(msg, lineno, line, self.filename) - - def error_no_section(self): - return self.parse_exc('Section must be started before assignment', - self.lineno) - - -class MultiConfigParser(object): - def __init__(self): - self.parsed = [] - - def read(self, config_files): - read_ok = [] - - for filename in config_files: - sections = {} - parser = ConfigParser(filename, sections) - - try: - parser.parse() - except IOError: - continue - self.parsed.insert(0, sections) - read_ok.append(filename) - - return read_ok - - def get(self, section, names, multi=False): - rvalue = [] - for sections in self.parsed: - if section not in sections: - continue - for name in names: - if name in sections[section]: - if multi: - rvalue = sections[section][name] + rvalue - else: - return sections[section][name] - if multi and rvalue != []: - return rvalue - raise KeyError - - -class ConfigOpts(collections.Mapping): - - """ - Config options which may be set on the command line or in config files. - - ConfigOpts is a configuration option manager with APIs for registering - option schemas, grouping options, parsing option values and retrieving - the values of options. - """ - - def __init__(self): - """Construct a ConfigOpts object.""" - self._opts = {} # dict of dicts of (opt:, override:, default:) - self._groups = {} - - self._args = None - self._oparser = None - self._cparser = None - self._cli_values = {} - self.__cache = {} - self._config_opts = [] - self._disable_interspersed_args = False - - def _setup(self, project, prog, version, usage, default_config_files): - """Initialize a ConfigOpts object for option parsing.""" - if prog is None: - prog = os.path.basename(sys.argv[0]) - - if default_config_files is None: - default_config_files = find_config_files(project, prog) - - self._oparser = optparse.OptionParser(prog=prog, - version=version, - usage=usage) - if self._disable_interspersed_args: - self._oparser.disable_interspersed_args() - - self._config_opts = [ - MultiStrOpt('config-file', - default=default_config_files, - metavar='PATH', - help='Path to a config file to use. Multiple config ' - 'files can be specified, with values in later ' - 'files taking precedence. The default files ' - ' used are: %s' % (default_config_files, )), - StrOpt('config-dir', - metavar='DIR', - help='Path to a config directory to pull *.conf ' - 'files from. This file set is sorted, so as to ' - 'provide a predictable parse order if individual ' - 'options are over-ridden. The set is parsed after ' - 'the file(s), if any, specified via --config-file, ' - 'hence over-ridden options in the directory take ' - 'precedence.'), - ] - self.register_cli_opts(self._config_opts) - - self.project = project - self.prog = prog - self.version = version - self.usage = usage - self.default_config_files = default_config_files - - def __clear_cache(f): - @functools.wraps(f) - def __inner(self, *args, **kwargs): - if kwargs.pop('clear_cache', True): - self.__cache.clear() - return f(self, *args, **kwargs) - - return __inner - - def __call__(self, - args=None, - project=None, - prog=None, - version=None, - usage=None, - default_config_files=None): - """Parse command line arguments and config files. - - Calling a ConfigOpts object causes the supplied command line arguments - and config files to be parsed, causing opt values to be made available - as attributes of the object. - - The object may be called multiple times, each time causing the previous - set of values to be overwritten. - - Automatically registers the --config-file option with either a supplied - list of default config files, or a list from find_config_files(). - - If the --config-dir option is set, any *.conf files from this - directory are pulled in, after all the file(s) specified by the - --config-file option. - - :param args: command line arguments (defaults to sys.argv[1:]) - :param project: the toplevel project name, used to locate config files - :param prog: the name of the program (defaults to sys.argv[0] basename) - :param version: the program version (for --version) - :param usage: a usage string (%prog will be expanded) - :param default_config_files: config files to use by default - :returns: the list of arguments left over after parsing options - :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError, - RequiredOptError, DuplicateOptError - """ - self.clear() - - self._setup(project, prog, version, usage, default_config_files) - - self._cli_values, leftovers = self._parse_cli_opts(args) - - self._parse_config_files() - - self._check_required_opts() - - return leftovers - - def __getattr__(self, name): - """Look up an option value and perform string substitution. - - :param name: the opt name (or 'dest', more precisely) - :returns: the option value (after string subsititution) or a GroupAttr - :raises: NoSuchOptError,ConfigFileValueError,TemplateSubstitutionError - """ - return self._get(name) - - def __getitem__(self, key): - """Look up an option value and perform string substitution.""" - return self.__getattr__(key) - - def __contains__(self, key): - """Return True if key is the name of a registered opt or group.""" - return key in self._opts or key in self._groups - - def __iter__(self): - """Iterate over all registered opt and group names.""" - for key in self._opts.keys() + self._groups.keys(): - yield key - - def __len__(self): - """Return the number of options and option groups.""" - return len(self._opts) + len(self._groups) - - def reset(self): - """Clear the object state and unset overrides and defaults.""" - self._unset_defaults_and_overrides() - self.clear() - - @__clear_cache - def clear(self): - """Clear the state of the object to before it was called.""" - self._args = None - self._cli_values.clear() - self._oparser = None - self._cparser = None - self.unregister_opts(self._config_opts) - for group in self._groups.values(): - group._clear() - - @__clear_cache - def register_opt(self, opt, group=None): - """Register an option schema. - - Registering an option schema makes any option value which is previously - or subsequently parsed from the command line or config files available - as an attribute of this object. - - :param opt: an instance of an Opt sub-class - :param group: an optional OptGroup object or group name - :return: False if the opt was already register, True otherwise - :raises: DuplicateOptError - """ - if group is not None: - return self._get_group(group, autocreate=True)._register_opt(opt) - - if _is_opt_registered(self._opts, opt): - return False - - self._opts[opt.dest] = {'opt': opt} - - return True - - @__clear_cache - def register_opts(self, opts, group=None): - """Register multiple option schemas at once.""" - for opt in opts: - self.register_opt(opt, group, clear_cache=False) - - @__clear_cache - def register_cli_opt(self, opt, group=None): - """Register a CLI option schema. - - CLI option schemas must be registered before the command line and - config files are parsed. This is to ensure that all CLI options are - show in --help and option validation works as expected. - - :param opt: an instance of an Opt sub-class - :param group: an optional OptGroup object or group name - :return: False if the opt was already register, True otherwise - :raises: DuplicateOptError, ArgsAlreadyParsedError - """ - if self._args is not None: - raise ArgsAlreadyParsedError("cannot register CLI option") - - return self.register_opt(opt, group, clear_cache=False) - - @__clear_cache - def register_cli_opts(self, opts, group=None): - """Register multiple CLI option schemas at once.""" - for opt in opts: - self.register_cli_opt(opt, group, clear_cache=False) - - def register_group(self, group): - """Register an option group. - - An option group must be registered before options can be registered - with the group. - - :param group: an OptGroup object - """ - if group.name in self._groups: - return - - self._groups[group.name] = copy.copy(group) - - @__clear_cache - def unregister_opt(self, opt, group=None): - """Unregister an option. - - :param opt: an Opt object - :param group: an optional OptGroup object or group name - :raises: ArgsAlreadyParsedError, NoSuchGroupError - """ - if self._args is not None: - raise ArgsAlreadyParsedError("reset before unregistering options") - - if group is not None: - self._get_group(group)._unregister_opt(opt) - elif opt.dest in self._opts: - del self._opts[opt.dest] - - @__clear_cache - def unregister_opts(self, opts, group=None): - """Unregister multiple CLI option schemas at once.""" - for opt in opts: - self.unregister_opt(opt, group, clear_cache=False) - - def import_opt(self, name, module_str, group=None): - """Import an option definition from a module. - - Import a module and check that a given option is registered. - - This is intended for use with global configuration objects - like cfg.CONF where modules commonly register options with - CONF at module load time. If one module requires an option - defined by another module it can use this method to explicitly - declare the dependency. - - :param name: the name/dest of the opt - :param module_str: the name of a module to import - :param group: an option OptGroup object or group name - :raises: NoSuchOptError, NoSuchGroupError - """ - __import__(module_str) - self._get_opt_info(name, group) - - @__clear_cache - def set_override(self, name, override, group=None): - """Override an opt value. - - Override the command line, config file and default values of a - given option. - - :param name: the name/dest of the opt - :param override: the override value - :param group: an option OptGroup object or group name - :raises: NoSuchOptError, NoSuchGroupError - """ - opt_info = self._get_opt_info(name, group) - opt_info['override'] = override - - @__clear_cache - def set_default(self, name, default, group=None): - """Override an opt's default value. - - Override the default value of given option. A command line or - config file value will still take precedence over this default. - - :param name: the name/dest of the opt - :param default: the default value - :param group: an option OptGroup object or group name - :raises: NoSuchOptError, NoSuchGroupError - """ - opt_info = self._get_opt_info(name, group) - opt_info['default'] = default - - @__clear_cache - def clear_override(self, name, group=None): - """Clear an override an opt value. - - Clear a previously set override of the command line, config file - and default values of a given option. - - :param name: the name/dest of the opt - :param group: an option OptGroup object or group name - :raises: NoSuchOptError, NoSuchGroupError - """ - opt_info = self._get_opt_info(name, group) - opt_info.pop('override', None) - - @__clear_cache - def clear_default(self, name, group=None): - """Clear an override an opt's default value. - - Clear a previously set override of the default value of given option. - - :param name: the name/dest of the opt - :param group: an option OptGroup object or group name - :raises: NoSuchOptError, NoSuchGroupError - """ - opt_info = self._get_opt_info(name, group) - opt_info.pop('default', None) - - def _all_opt_infos(self): - """A generator function for iteration opt infos.""" - for info in self._opts.values(): - yield info, None - for group in self._groups.values(): - for info in group._opts.values(): - yield info, group - - def _all_opts(self): - """A generator function for iteration opts.""" - for info, group in self._all_opt_infos(): - yield info['opt'], group - - def _unset_defaults_and_overrides(self): - """Unset any default or override on all options.""" - for info, group in self._all_opt_infos(): - info.pop('default', None) - info.pop('override', None) - - def disable_interspersed_args(self): - """Set parsing to stop on the first non-option. - - If this this method is called, then parsing e.g. - - script --verbose cmd --debug /tmp/mything - - will no longer return: - - ['cmd', '/tmp/mything'] - - as the leftover arguments, but will instead return: - - ['cmd', '--debug', '/tmp/mything'] - - i.e. argument parsing is stopped at the first non-option argument. - """ - self._disable_interspersed_args = True - - def enable_interspersed_args(self): - """Set parsing to not stop on the first non-option. - - This it the default behaviour.""" - self._disable_interspersed_args = False - - def find_file(self, name): - """Locate a file located alongside the config files. - - Search for a file with the supplied basename in the directories - which we have already loaded config files from and other known - configuration directories. - - The directory, if any, supplied by the config_dir option is - searched first. Then the config_file option is iterated over - and each of the base directories of the config_files values - are searched. Failing both of these, the standard directories - searched by the module level find_config_files() function is - used. The first matching file is returned. - - :param basename: the filename, e.g. 'policy.json' - :returns: the path to a matching file, or None - """ - dirs = [] - if self.config_dir: - dirs.append(_fixpath(self.config_dir)) - - for cf in reversed(self.config_file): - dirs.append(os.path.dirname(_fixpath(cf))) - - dirs.extend(_get_config_dirs(self.project)) - - return _search_dirs(dirs, name) - - def log_opt_values(self, logger, lvl): - """Log the value of all registered opts. - - It's often useful for an app to log its configuration to a log file at - startup for debugging. This method dumps to the entire config state to - the supplied logger at a given log level. - - :param logger: a logging.Logger object - :param lvl: the log level (e.g. logging.DEBUG) arg to logger.log() - """ - logger.log(lvl, "*" * 80) - logger.log(lvl, "Configuration options gathered from:") - logger.log(lvl, "command line args: %s", self._args) - logger.log(lvl, "config files: %s", self.config_file) - logger.log(lvl, "=" * 80) - - def _sanitize(opt, value): - """Obfuscate values of options declared secret""" - return value if not opt.secret else '*' * len(str(value)) - - for opt_name in sorted(self._opts): - opt = self._get_opt_info(opt_name)['opt'] - logger.log(lvl, "%-30s = %s", opt_name, - _sanitize(opt, getattr(self, opt_name))) - - for group_name in self._groups: - group_attr = self.GroupAttr(self, self._get_group(group_name)) - for opt_name in sorted(self._groups[group_name]._opts): - opt = self._get_opt_info(opt_name, group_name)['opt'] - logger.log(lvl, "%-30s = %s", - "%s.%s" % (group_name, opt_name), - _sanitize(opt, getattr(group_attr, opt_name))) - - logger.log(lvl, "*" * 80) - - def print_usage(self, file=None): - """Print the usage message for the current program.""" - self._oparser.print_usage(file) - - def print_help(self, file=None): - """Print the help message for the current program.""" - self._oparser.print_help(file) - - def _get(self, name, group=None): - if isinstance(group, OptGroup): - key = (group.name, name) - else: - key = (group, name) - try: - return self.__cache[key] - except KeyError: - value = self._substitute(self._do_get(name, group)) - self.__cache[key] = value - return value - - def _do_get(self, name, group=None): - """Look up an option value. - - :param name: the opt name (or 'dest', more precisely) - :param group: an OptGroup - :returns: the option value, or a GroupAttr object - :raises: NoSuchOptError, NoSuchGroupError, ConfigFileValueError, - TemplateSubstitutionError - """ - if group is None and name in self._groups: - return self.GroupAttr(self, self._get_group(name)) - - info = self._get_opt_info(name, group) - opt = info['opt'] - - if 'override' in info: - return info['override'] - - values = [] - if self._cparser is not None: - section = group.name if group is not None else 'DEFAULT' - try: - value = opt._get_from_config_parser(self._cparser, section) - except KeyError: - pass - except ValueError as ve: - raise ConfigFileValueError(str(ve)) - else: - if not opt.multi: - # No need to continue since the last value wins - return value[-1] - values.extend(value) - - name = name if group is None else group.name + '_' + name - value = self._cli_values.get(name) - if value is not None: - if not opt.multi: - return value - - return value + values - - if values: - return values - - if 'default' in info: - return info['default'] - - return opt.default - - def _substitute(self, value): - """Perform string template substitution. - - Substitute any template variables (e.g. $foo, ${bar}) in the supplied - string value(s) with opt values. - - :param value: the string value, or list of string values - :returns: the substituted string(s) - """ - if isinstance(value, list): - return [self._substitute(i) for i in value] - elif isinstance(value, str): - tmpl = string.Template(value) - return tmpl.safe_substitute(self.StrSubWrapper(self)) - else: - return value - - def _get_group(self, group_or_name, autocreate=False): - """Looks up a OptGroup object. - - Helper function to return an OptGroup given a parameter which can - either be the group's name or an OptGroup object. - - The OptGroup object returned is from the internal dict of OptGroup - objects, which will be a copy of any OptGroup object that users of - the API have access to. - - :param group_or_name: the group's name or the OptGroup object itself - :param autocreate: whether to auto-create the group if it's not found - :raises: NoSuchGroupError - """ - group = group_or_name if isinstance(group_or_name, OptGroup) else None - group_name = group.name if group else group_or_name - - if not group_name in self._groups: - if not group is None or not autocreate: - raise NoSuchGroupError(group_name) - - self.register_group(OptGroup(name=group_name)) - - return self._groups[group_name] - - def _get_opt_info(self, opt_name, group=None): - """Return the (opt, override, default) dict for an opt. - - :param opt_name: an opt name/dest - :param group: an optional group name or OptGroup object - :raises: NoSuchOptError, NoSuchGroupError - """ - if group is None: - opts = self._opts - else: - group = self._get_group(group) - opts = group._opts - - if not opt_name in opts: - raise NoSuchOptError(opt_name, group) - - return opts[opt_name] - - def _parse_config_files(self): - """Parse the config files from --config-file and --config-dir. - - :raises: ConfigFilesNotFoundError, ConfigFileParseError - """ - config_files = list(self.config_file) - - if self.config_dir: - config_dir_glob = os.path.join(self.config_dir, '*.conf') - config_files += sorted(glob.glob(config_dir_glob)) - - config_files = [_fixpath(p) for p in config_files] - - self._cparser = MultiConfigParser() - - try: - read_ok = self._cparser.read(config_files) - except iniparser.ParseError as pe: - raise ConfigFileParseError(pe.filename, str(pe)) - - if read_ok != config_files: - not_read_ok = filter(lambda f: f not in read_ok, config_files) - raise ConfigFilesNotFoundError(not_read_ok) - - def _check_required_opts(self): - """Check that all opts marked as required have values specified. - - :raises: RequiredOptError - """ - for info, group in self._all_opt_infos(): - opt = info['opt'] - - if opt.required: - if ('default' in info or 'override' in info): - continue - - if self._get(opt.dest, group) is None: - raise RequiredOptError(opt.name, group) - - def _parse_cli_opts(self, args): - """Parse command line options. - - Initializes the command line option parser and parses the supplied - command line arguments. - - :param args: the command line arguments - :returns: a dict of parsed option values - :raises: SystemExit, DuplicateOptError - - """ - self._args = args - - for opt, group in self._all_opts(): - opt._add_to_cli(self._oparser, group) - - values, leftovers = self._oparser.parse_args(args) - - return vars(values), leftovers - - class GroupAttr(collections.Mapping): - - """ - A helper class representing the option values of a group as a mapping - and attributes. - """ - - def __init__(self, conf, group): - """Construct a GroupAttr object. - - :param conf: a ConfigOpts object - :param group: an OptGroup object - """ - self.conf = conf - self.group = group - - def __getattr__(self, name): - """Look up an option value and perform template substitution.""" - return self.conf._get(name, self.group) - - def __getitem__(self, key): - """Look up an option value and perform string substitution.""" - return self.__getattr__(key) - - def __contains__(self, key): - """Return True if key is the name of a registered opt or group.""" - return key in self.group._opts - - def __iter__(self): - """Iterate over all registered opt and group names.""" - for key in self.group._opts.keys(): - yield key - - def __len__(self): - """Return the number of options and option groups.""" - return len(self.group._opts) - - class StrSubWrapper(object): - - """ - A helper class exposing opt values as a dict for string substitution. - """ - - def __init__(self, conf): - """Construct a StrSubWrapper object. - - :param conf: a ConfigOpts object - """ - self.conf = conf - - def __getitem__(self, key): - """Look up an opt value from the ConfigOpts object. - - :param key: an opt name - :returns: an opt value - :raises: TemplateSubstitutionError if attribute is a group - """ - value = getattr(self.conf, key) - if isinstance(value, self.conf.GroupAttr): - raise TemplateSubstitutionError( - 'substituting group %s not supported' % key) - return value - - -class CommonConfigOpts(ConfigOpts): - - DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s" - DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" - - common_cli_opts = [ - BoolOpt('debug', - short='d', - default=False, - help='Print debugging output'), - BoolOpt('verbose', - short='v', - default=False, - help='Print more verbose output'), - ] - - logging_cli_opts = [ - StrOpt('log-config', - metavar='PATH', - help='If this option is specified, the logging configuration ' - 'file specified is used and overrides any other logging ' - 'options specified. Please see the Python logging module ' - 'documentation for details on logging configuration ' - 'files.'), - StrOpt('log-format', - default=DEFAULT_LOG_FORMAT, - metavar='FORMAT', - help='A logging.Formatter log message format string which may ' - 'use any of the available logging.LogRecord attributes. ' - 'Default: %default'), - StrOpt('log-date-format', - default=DEFAULT_LOG_DATE_FORMAT, - metavar='DATE_FORMAT', - help='Format string for %(asctime)s in log records. ' - 'Default: %default'), - StrOpt('log-file', - metavar='PATH', - help='(Optional) Name of log file to output to. ' - 'If not set, logging will go to stdout.'), - StrOpt('log-dir', - help='(Optional) The directory to keep log files in ' - '(will be prepended to --logfile)'), - BoolOpt('use-syslog', - default=False, - help='Use syslog for logging.'), - StrOpt('syslog-log-facility', - default='LOG_USER', - help='syslog facility to receive log lines') - ] - - def __init__(self): - super(CommonConfigOpts, self).__init__() - self.register_cli_opts(self.common_cli_opts) - self.register_cli_opts(self.logging_cli_opts) - - -CONF = CommonConfigOpts() diff --git a/bufunfa/openstack/common/context.py b/bufunfa/openstack/common/context.py deleted file mode 100644 index dd7dd04..0000000 --- a/bufunfa/openstack/common/context.py +++ /dev/null @@ -1,81 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Simple class that stores security context information in the web request. - -Projects should subclass this class if they wish to enhance the request -context or provide additional information in their specific WSGI pipeline. -""" - -import itertools -import uuid - - -def generate_request_id(): - return 'req-' + str(uuid.uuid4()) - - -class RequestContext(object): - - """ - Stores information about the security context under which the user - accesses the system, as well as additional request information. - """ - - def __init__(self, auth_tok=None, user=None, tenant=None, is_admin=False, - read_only=False, show_deleted=False, request_id=None): - self.auth_tok = auth_tok - self.user = user - self.tenant = tenant - self.is_admin = is_admin - self.read_only = read_only - self.show_deleted = show_deleted - if not request_id: - request_id = generate_request_id() - self.request_id = request_id - - def to_dict(self): - return {'user': self.user, - 'tenant': self.tenant, - 'is_admin': self.is_admin, - 'read_only': self.read_only, - 'show_deleted': self.show_deleted, - 'auth_token': self.auth_tok, - 'request_id': self.request_id} - - -def get_admin_context(show_deleted="no"): - context = RequestContext(None, - tenant=None, - is_admin=True, - show_deleted=show_deleted) - return context - - -def get_context_from_function_and_args(function, args, kwargs): - """Find an arg of type RequestContext and return it. - - This is useful in a couple of decorators where we don't - know much about the function we're wrapping. - """ - - for arg in itertools.chain(kwargs.values(), args): - if isinstance(arg, RequestContext): - return arg - - return None diff --git a/bufunfa/openstack/common/eventlet_backdoor.py b/bufunfa/openstack/common/eventlet_backdoor.py deleted file mode 100644 index 27415cb..0000000 --- a/bufunfa/openstack/common/eventlet_backdoor.py +++ /dev/null @@ -1,78 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2012 Openstack, LLC. -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import gc -import pprint -import sys -import traceback - -import eventlet -import eventlet.backdoor -import greenlet - -from bufunfa.openstack.common import cfg - -eventlet_backdoor_opts = [ - cfg.IntOpt('backdoor_port', - default=None, - help='port for eventlet backdoor to listen') -] - -CONF = cfg.CONF -CONF.register_opts(eventlet_backdoor_opts) - - -def _dont_use_this(): - print "Don't use this, just disconnect instead" - - -def _find_objects(t): - return filter(lambda o: isinstance(o, t), gc.get_objects()) - - -def _print_greenthreads(): - for i, gt in enumerate(find_objects(greenlet.greenlet)): - print i, gt - traceback.print_stack(gt.gr_frame) - print - - -def initialize_if_enabled(): - backdoor_locals = { - 'exit': _dont_use_this, # So we don't exit the entire process - 'quit': _dont_use_this, # So we don't exit the entire process - 'fo': _find_objects, - 'pgt': _print_greenthreads, - } - - if CONF.backdoor_port is None: - return - - # NOTE(johannes): The standard sys.displayhook will print the value of - # the last expression and set it to __builtin__._, which overwrites - # the __builtin__._ that gettext sets. Let's switch to using pprint - # since it won't interact poorly with gettext, and it's easier to - # read the output too. - def displayhook(val): - if val is not None: - pprint.pprint(val) - sys.displayhook = displayhook - - eventlet.spawn(eventlet.backdoor.backdoor_server, - eventlet.listen(('localhost', CONF.backdoor_port)), - locals=backdoor_locals) diff --git a/bufunfa/openstack/common/exception.py b/bufunfa/openstack/common/exception.py deleted file mode 100644 index 4866de2..0000000 --- a/bufunfa/openstack/common/exception.py +++ /dev/null @@ -1,135 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Exceptions common to OpenStack projects -""" - -import logging - - -class Error(Exception): - def __init__(self, message=None): - super(Error, self).__init__(message) - - -class ApiError(Error): - def __init__(self, message='Unknown', code='Unknown'): - self.message = message - self.code = code - super(ApiError, self).__init__('%s: %s' % (code, message)) - - -class NotFound(Error): - pass - - -class UnknownScheme(Error): - - msg = "Unknown scheme '%s' found in URI" - - def __init__(self, scheme): - msg = self.__class__.msg % scheme - super(UnknownScheme, self).__init__(msg) - - -class BadStoreUri(Error): - - msg = "The Store URI %s was malformed. Reason: %s" - - def __init__(self, uri, reason): - msg = self.__class__.msg % (uri, reason) - super(BadStoreUri, self).__init__(msg) - - -class Duplicate(Error): - pass - - -class NotAuthorized(Error): - pass - - -class NotEmpty(Error): - pass - - -class Invalid(Error): - pass - - -class BadInputError(Exception): - """Error resulting from a client sending bad input to a server""" - pass - - -class MissingArgumentError(Error): - pass - - -class DatabaseMigrationError(Error): - pass - - -class ClientConnectionError(Exception): - """Error resulting from a client connecting to a server""" - pass - - -def wrap_exception(f): - def _wrap(*args, **kw): - try: - return f(*args, **kw) - except Exception, e: - if not isinstance(e, Error): - #exc_type, exc_value, exc_traceback = sys.exc_info() - logging.exception('Uncaught exception') - #logging.error(traceback.extract_stack(exc_traceback)) - raise Error(str(e)) - raise - _wrap.func_name = f.func_name - return _wrap - - -class OpenstackException(Exception): - """ - Base Exception - - To correctly use this class, inherit from it and define - a 'message' property. That message will get printf'd - with the keyword arguments provided to the constructor. - """ - message = "An unknown exception occurred" - - def __init__(self, **kwargs): - try: - self._error_string = self.message % kwargs - - except Exception: - # at least get the core message out if something happened - self._error_string = self.message - - def __str__(self): - return self._error_string - - -class MalformedRequestBody(OpenstackException): - message = "Malformed message body: %(reason)s" - - -class InvalidContentType(OpenstackException): - message = "Invalid content type %(content_type)s" diff --git a/bufunfa/openstack/common/excutils.py b/bufunfa/openstack/common/excutils.py deleted file mode 100644 index 5dd4830..0000000 --- a/bufunfa/openstack/common/excutils.py +++ /dev/null @@ -1,49 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# Copyright 2012, Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Exception related utilities. -""" - -import contextlib -import logging -import sys -import traceback - - -@contextlib.contextmanager -def save_and_reraise_exception(): - """Save current exception, run some code and then re-raise. - - In some cases the exception context can be cleared, resulting in None - being attempted to be re-raised after an exception handler is run. This - can happen when eventlet switches greenthreads or when running an - exception handler, code raises and catches an exception. In both - cases the exception context will be cleared. - - To work around this, we save the exception state, run handler code, and - then re-raise the original exception. If another exception occurs, the - saved exception is logged and the new exception is re-raised. - """ - type_, value, tb = sys.exc_info() - try: - yield - except Exception: - logging.error('Original exception being dropped: %s' % - (traceback.format_exception(type_, value, tb))) - raise - raise type_, value, tb diff --git a/bufunfa/openstack/common/gettextutils.py b/bufunfa/openstack/common/gettextutils.py deleted file mode 100644 index 235350c..0000000 --- a/bufunfa/openstack/common/gettextutils.py +++ /dev/null @@ -1,33 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 Red Hat, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -gettext for openstack-common modules. - -Usual usage in an openstack.common module: - - from openstack.common.gettextutils import _ -""" - -import gettext - - -t = gettext.translation('openstack-common', 'locale', fallback=True) - - -def _(msg): - return t.ugettext(msg) diff --git a/bufunfa/openstack/common/importutils.py b/bufunfa/openstack/common/importutils.py deleted file mode 100644 index f45372b..0000000 --- a/bufunfa/openstack/common/importutils.py +++ /dev/null @@ -1,59 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Import related utilities and helper functions. -""" - -import sys -import traceback - - -def import_class(import_str): - """Returns a class from a string including module and class""" - mod_str, _sep, class_str = import_str.rpartition('.') - try: - __import__(mod_str) - return getattr(sys.modules[mod_str], class_str) - except (ValueError, AttributeError), exc: - raise ImportError('Class %s cannot be found (%s)' % - (class_str, - traceback.format_exception(*sys.exc_info()))) - - -def import_object(import_str, *args, **kwargs): - """Import a class and return an instance of it.""" - return import_class(import_str)(*args, **kwargs) - - -def import_object_ns(name_space, import_str, *args, **kwargs): - """ - Import a class and return an instance of it, first by trying - to find the class in a default namespace, then failing back to - a full path if not found in the default namespace. - """ - import_value = "%s.%s" % (name_space, import_str) - try: - return import_class(import_value)(*args, **kwargs) - except ImportError: - return import_class(import_str)(*args, **kwargs) - - -def import_module(import_str): - """Import a module.""" - __import__(import_str) - return sys.modules[import_str] diff --git a/bufunfa/openstack/common/iniparser.py b/bufunfa/openstack/common/iniparser.py deleted file mode 100644 index 2412844..0000000 --- a/bufunfa/openstack/common/iniparser.py +++ /dev/null @@ -1,130 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 OpenStack LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class ParseError(Exception): - def __init__(self, message, lineno, line): - self.msg = message - self.line = line - self.lineno = lineno - - def __str__(self): - return 'at line %d, %s: %r' % (self.lineno, self.msg, self.line) - - -class BaseParser(object): - lineno = 0 - parse_exc = ParseError - - def _assignment(self, key, value): - self.assignment(key, value) - return None, [] - - def _get_section(self, line): - if line[-1] != ']': - return self.error_no_section_end_bracket(line) - if len(line) <= 2: - return self.error_no_section_name(line) - - return line[1:-1] - - def _split_key_value(self, line): - colon = line.find(':') - equal = line.find('=') - if colon < 0 and equal < 0: - return self.error_invalid_assignment(line) - - if colon < 0 or (equal >= 0 and equal < colon): - key, value = line[:equal], line[equal + 1:] - else: - key, value = line[:colon], line[colon + 1:] - - value = value.strip() - if ((value and value[0] == value[-1]) and - (value[0] == "\"" or value[0] == "'")): - value = value[1:-1] - return key.strip(), [value] - - def parse(self, lineiter): - key = None - value = [] - - for line in lineiter: - self.lineno += 1 - - line = line.rstrip() - if not line: - # Blank line, ends multi-line values - if key: - key, value = self._assignment(key, value) - continue - elif line[0] in (' ', '\t'): - # Continuation of previous assignment - if key is None: - self.error_unexpected_continuation(line) - else: - value.append(line.lstrip()) - continue - - if key: - # Flush previous assignment, if any - key, value = self._assignment(key, value) - - if line[0] == '[': - # Section start - section = self._get_section(line) - if section: - self.new_section(section) - elif line[0] in '#;': - self.comment(line[1:].lstrip()) - else: - key, value = self._split_key_value(line) - if not key: - return self.error_empty_key(line) - - if key: - # Flush previous assignment, if any - self._assignment(key, value) - - def assignment(self, key, value): - """Called when a full assignment is parsed""" - raise NotImplementedError() - - def new_section(self, section): - """Called when a new section is started""" - raise NotImplementedError() - - def comment(self, comment): - """Called when a comment is parsed""" - pass - - def error_invalid_assignment(self, line): - raise self.parse_exc("No ':' or '=' found in assignment", - self.lineno, line) - - def error_empty_key(self, line): - raise self.parse_exc('Key cannot be empty', self.lineno, line) - - def error_unexpected_continuation(self, line): - raise self.parse_exc('Unexpected continuation line', - self.lineno, line) - - def error_no_section_end_bracket(self, line): - raise self.parse_exc('Invalid section (must end with ])', - self.lineno, line) - - def error_no_section_name(self, line): - raise self.parse_exc('Empty section name', self.lineno, line) diff --git a/bufunfa/openstack/common/jsonutils.py b/bufunfa/openstack/common/jsonutils.py deleted file mode 100644 index 48dc261..0000000 --- a/bufunfa/openstack/common/jsonutils.py +++ /dev/null @@ -1,148 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -''' -JSON related utilities. - -This module provides a few things: - - 1) A handy function for getting an object down to something that can be - JSON serialized. See to_primitive(). - - 2) Wrappers around loads() and dumps(). The dumps() wrapper will - automatically use to_primitive() for you if needed. - - 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson - is available. -''' - - -import datetime -import inspect -import itertools -import json -import xmlrpclib - -from bufunfa.openstack.common import timeutils - - -def to_primitive(value, convert_instances=False, level=0): - """Convert a complex object into primitives. - - Handy for JSON serialization. We can optionally handle instances, - but since this is a recursive function, we could have cyclical - data structures. - - To handle cyclical data structures we could track the actual objects - visited in a set, but not all objects are hashable. Instead we just - track the depth of the object inspections and don't go too deep. - - Therefore, convert_instances=True is lossy ... be aware. - - """ - nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod, - inspect.isfunction, inspect.isgeneratorfunction, - inspect.isgenerator, inspect.istraceback, inspect.isframe, - inspect.iscode, inspect.isbuiltin, inspect.isroutine, - inspect.isabstract] - for test in nasty: - if test(value): - return unicode(value) - - # value of itertools.count doesn't get caught by inspects - # above and results in infinite loop when list(value) is called. - if type(value) == itertools.count: - return unicode(value) - - # FIXME(vish): Workaround for LP bug 852095. Without this workaround, - # tests that raise an exception in a mocked method that - # has a @wrap_exception with a notifier will fail. If - # we up the dependency to 0.5.4 (when it is released) we - # can remove this workaround. - if getattr(value, '__module__', None) == 'mox': - return 'mock' - - if level > 3: - return '?' - - # The try block may not be necessary after the class check above, - # but just in case ... - try: - # It's not clear why xmlrpclib created their own DateTime type, but - # for our purposes, make it a datetime type which is explicitly - # handled - if isinstance(value, xmlrpclib.DateTime): - value = datetime.datetime(*tuple(value.timetuple())[:6]) - - if isinstance(value, (list, tuple)): - o = [] - for v in value: - o.append(to_primitive(v, convert_instances=convert_instances, - level=level)) - return o - elif isinstance(value, dict): - o = {} - for k, v in value.iteritems(): - o[k] = to_primitive(v, convert_instances=convert_instances, - level=level) - return o - elif isinstance(value, datetime.datetime): - return timeutils.strtime(value) - elif hasattr(value, 'iteritems'): - return to_primitive(dict(value.iteritems()), - convert_instances=convert_instances, - level=level + 1) - elif hasattr(value, '__iter__'): - return to_primitive(list(value), - convert_instances=convert_instances, - level=level) - elif convert_instances and hasattr(value, '__dict__'): - # Likely an instance of something. Watch for cycles. - # Ignore class member vars. - return to_primitive(value.__dict__, - convert_instances=convert_instances, - level=level + 1) - else: - return value - except TypeError, e: - # Class objects are tricky since they may define something like - # __iter__ defined but it isn't callable as list(). - return unicode(value) - - -def dumps(value, default=to_primitive, **kwargs): - return json.dumps(value, default=default, **kwargs) - - -def loads(s): - return json.loads(s) - - -def load(s): - return json.load(s) - - -try: - import anyjson -except ImportError: - pass -else: - anyjson._modules.append((__name__, 'dumps', TypeError, - 'loads', ValueError, 'load')) - anyjson.force_implementation(__name__) diff --git a/bufunfa/openstack/common/local.py b/bufunfa/openstack/common/local.py deleted file mode 100644 index 19d9627..0000000 --- a/bufunfa/openstack/common/local.py +++ /dev/null @@ -1,37 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Greenthread local storage of variables using weak references""" - -import weakref - -from eventlet import corolocal - - -class WeakLocal(corolocal.local): - def __getattribute__(self, attr): - rval = corolocal.local.__getattribute__(self, attr) - if rval: - rval = rval() - return rval - - def __setattr__(self, attr, value): - value = weakref.ref(value) - return corolocal.local.__setattr__(self, attr, value) - - -store = WeakLocal() diff --git a/bufunfa/openstack/common/log.py b/bufunfa/openstack/common/log.py deleted file mode 100644 index 6202a93..0000000 --- a/bufunfa/openstack/common/log.py +++ /dev/null @@ -1,470 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Openstack logging handler. - -This module adds to logging functionality by adding the option to specify -a context object when calling the various log methods. If the context object -is not specified, default formatting is used. Additionally, an instance uuid -may be passed as part of the log message, which is intended to make it easier -for admins to find messages related to a specific instance. - -It also allows setting of formatting information through conf. - -""" - -import cStringIO -import inspect -import itertools -import logging -import logging.config -import logging.handlers -import os -import stat -import sys -import traceback - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import jsonutils -from bufunfa.openstack.common import local -from bufunfa.openstack.common import notifier - - -log_opts = [ - cfg.StrOpt('logging_context_format_string', - default='%(asctime)s %(levelname)s %(name)s [%(request_id)s ' - '%(user_id)s %(project_id)s] %(instance)s' - '%(message)s', - help='format string to use for log messages with context'), - cfg.StrOpt('logging_default_format_string', - default='%(asctime)s %(process)d %(levelname)s %(name)s [-]' - ' %(instance)s%(message)s', - help='format string to use for log messages without context'), - cfg.StrOpt('logging_debug_format_suffix', - default='%(funcName)s %(pathname)s:%(lineno)d', - help='data to append to log format when level is DEBUG'), - cfg.StrOpt('logging_exception_prefix', - default='%(asctime)s %(process)d TRACE %(name)s %(instance)s', - help='prefix each line of exception output with this format'), - cfg.ListOpt('default_log_levels', - default=[ - 'amqplib=WARN', - 'sqlalchemy=WARN', - 'boto=WARN', - 'suds=INFO', - 'keystone=INFO', - 'eventlet.wsgi.server=WARN' - ], - help='list of logger=LEVEL pairs'), - cfg.BoolOpt('publish_errors', - default=False, - help='publish error events'), - cfg.BoolOpt('fatal_deprecations', - default=False, - help='make deprecations fatal'), - - # NOTE(mikal): there are two options here because sometimes we are handed - # a full instance (and could include more information), and other times we - # are just handed a UUID for the instance. - cfg.StrOpt('instance_format', - default='[instance: %(uuid)s] ', - help='If an instance is passed with the log message, format ' - 'it like this'), - cfg.StrOpt('instance_uuid_format', - default='[instance: %(uuid)s] ', - help='If an instance UUID is passed with the log message, ' - 'format it like this'), -] - - -generic_log_opts = [ - cfg.StrOpt('logdir', - default=None, - help='Log output to a per-service log file in named directory'), - cfg.StrOpt('logfile', - default=None, - help='Log output to a named file'), - cfg.BoolOpt('use_stderr', - default=True, - help='Log output to standard error'), - cfg.StrOpt('logfile_mode', - default='0644', - help='Default file mode used when creating log files'), -] - - -CONF = cfg.CONF -CONF.register_opts(generic_log_opts) -CONF.register_opts(log_opts) - -# our new audit level -# NOTE(jkoelker) Since we synthesized an audit level, make the logging -# module aware of it so it acts like other levels. -logging.AUDIT = logging.INFO + 1 -logging.addLevelName(logging.AUDIT, 'AUDIT') - - -try: - NullHandler = logging.NullHandler -except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7 - class NullHandler(logging.Handler): - def handle(self, record): - pass - - def emit(self, record): - pass - - def createLock(self): - self.lock = None - - -def _dictify_context(context): - if context is None: - return None - if not isinstance(context, dict) and getattr(context, 'to_dict', None): - context = context.to_dict() - return context - - -def _get_binary_name(): - return os.path.basename(inspect.stack()[-1][1]) - - -def _get_log_file_path(binary=None): - logfile = CONF.log_file or CONF.logfile - logdir = CONF.log_dir or CONF.logdir - - if logfile and not logdir: - return logfile - - if logfile and logdir: - return os.path.join(logdir, logfile) - - if logdir: - binary = binary or _get_binary_name() - return '%s.log' % (os.path.join(logdir, binary),) - - -class ContextAdapter(logging.LoggerAdapter): - warn = logging.LoggerAdapter.warning - - def __init__(self, logger, project_name, version_string): - self.logger = logger - self.project = project_name - self.version = version_string - - def audit(self, msg, *args, **kwargs): - self.log(logging.AUDIT, msg, *args, **kwargs) - - def deprecated(self, msg, *args, **kwargs): - stdmsg = _("Deprecated Config: %s") % msg - if CONF.fatal_deprecations: - self.critical(stdmsg, *args, **kwargs) - raise DeprecatedConfig(msg=stdmsg) - else: - self.warn(stdmsg, *args, **kwargs) - - def process(self, msg, kwargs): - if 'extra' not in kwargs: - kwargs['extra'] = {} - extra = kwargs['extra'] - - context = kwargs.pop('context', None) - if not context: - context = getattr(local.store, 'context', None) - if context: - extra.update(_dictify_context(context)) - - instance = kwargs.pop('instance', None) - instance_extra = '' - if instance: - instance_extra = CONF.instance_format % instance - else: - instance_uuid = kwargs.pop('instance_uuid', None) - if instance_uuid: - instance_extra = (CONF.instance_uuid_format - % {'uuid': instance_uuid}) - extra.update({'instance': instance_extra}) - - extra.update({"project": self.project}) - extra.update({"version": self.version}) - extra['extra'] = extra.copy() - return msg, kwargs - - -class JSONFormatter(logging.Formatter): - def __init__(self, fmt=None, datefmt=None): - # NOTE(jkoelker) we ignore the fmt argument, but its still there - # since logging.config.fileConfig passes it. - self.datefmt = datefmt - - def formatException(self, ei, strip_newlines=True): - lines = traceback.format_exception(*ei) - if strip_newlines: - lines = [itertools.ifilter( - lambda x: x, - line.rstrip().splitlines()) for line in lines] - lines = list(itertools.chain(*lines)) - return lines - - def format(self, record): - message = {'message': record.getMessage(), - 'asctime': self.formatTime(record, self.datefmt), - 'name': record.name, - 'msg': record.msg, - 'args': record.args, - 'levelname': record.levelname, - 'levelno': record.levelno, - 'pathname': record.pathname, - 'filename': record.filename, - 'module': record.module, - 'lineno': record.lineno, - 'funcname': record.funcName, - 'created': record.created, - 'msecs': record.msecs, - 'relative_created': record.relativeCreated, - 'thread': record.thread, - 'thread_name': record.threadName, - 'process_name': record.processName, - 'process': record.process, - 'traceback': None} - - if hasattr(record, 'extra'): - message['extra'] = record.extra - - if record.exc_info: - message['traceback'] = self.formatException(record.exc_info) - - return jsonutils.dumps(message) - - -class PublishErrorsHandler(logging.Handler): - def emit(self, record): - if ('bufunfa.openstack.common.notifier.log_notifier' in - CONF.notification_driver): - return - notifier.api.notify(None, 'error.publisher', - 'error_notification', - notifier.api.ERROR, - dict(error=record.msg)) - - -def _create_logging_excepthook(product_name): - def logging_excepthook(type, value, tb): - extra = {} - if CONF.verbose: - extra['exc_info'] = (type, value, tb) - getLogger(product_name).critical(str(value), **extra) - return logging_excepthook - - -def setup(product_name): - """Setup logging.""" - sys.excepthook = _create_logging_excepthook(product_name) - - if CONF.log_config: - try: - logging.config.fileConfig(CONF.log_config) - except Exception: - traceback.print_exc() - raise - else: - _setup_logging_from_conf(product_name) - - -def _find_facility_from_conf(): - facility_names = logging.handlers.SysLogHandler.facility_names - facility = getattr(logging.handlers.SysLogHandler, - CONF.syslog_log_facility, - None) - - if facility is None and CONF.syslog_log_facility in facility_names: - facility = facility_names.get(CONF.syslog_log_facility) - - if facility is None: - valid_facilities = facility_names.keys() - consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON', - 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS', - 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP', - 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3', - 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7'] - valid_facilities.extend(consts) - raise TypeError(_('syslog facility must be one of: %s') % - ', '.join("'%s'" % fac - for fac in valid_facilities)) - - return facility - - -def _setup_logging_from_conf(product_name): - log_root = getLogger(product_name).logger - for handler in log_root.handlers: - log_root.removeHandler(handler) - - if CONF.use_syslog: - facility = _find_facility_from_conf() - syslog = logging.handlers.SysLogHandler(address='/dev/log', - facility=facility) - log_root.addHandler(syslog) - - logpath = _get_log_file_path() - if logpath: - filelog = logging.handlers.WatchedFileHandler(logpath) - log_root.addHandler(filelog) - - mode = int(CONF.logfile_mode, 8) - st = os.stat(logpath) - if st.st_mode != (stat.S_IFREG | mode): - os.chmod(logpath, mode) - - if CONF.use_stderr: - streamlog = ColorHandler() - log_root.addHandler(streamlog) - - elif not CONF.log_file: - # pass sys.stdout as a positional argument - # python2.6 calls the argument strm, in 2.7 it's stream - streamlog = logging.StreamHandler(sys.stdout) - log_root.addHandler(streamlog) - - if CONF.publish_errors: - log_root.addHandler(PublishErrorsHandler(logging.ERROR)) - - for handler in log_root.handlers: - datefmt = CONF.log_date_format - if CONF.log_format: - handler.setFormatter(logging.Formatter(fmt=CONF.log_format, - datefmt=datefmt)) - handler.setFormatter(LegacyFormatter(datefmt=datefmt)) - - if CONF.verbose or CONF.debug: - log_root.setLevel(logging.DEBUG) - else: - log_root.setLevel(logging.INFO) - - level = logging.NOTSET - for pair in CONF.default_log_levels: - mod, _sep, level_name = pair.partition('=') - level = logging.getLevelName(level_name) - logger = logging.getLogger(mod) - logger.setLevel(level) - for handler in log_root.handlers: - logger.addHandler(handler) - -_loggers = {} - - -def getLogger(name='unknown', version='unknown'): - if name not in _loggers: - _loggers[name] = ContextAdapter(logging.getLogger(name), - name, - version) - return _loggers[name] - - -class WritableLogger(object): - """A thin wrapper that responds to `write` and logs.""" - - def __init__(self, logger, level=logging.INFO): - self.logger = logger - self.level = level - - def write(self, msg): - self.logger.log(self.level, msg) - - -class LegacyFormatter(logging.Formatter): - """A context.RequestContext aware formatter configured through flags. - - The flags used to set format strings are: logging_context_format_string - and logging_default_format_string. You can also specify - logging_debug_format_suffix to append extra formatting if the log level is - debug. - - For information about what variables are available for the formatter see: - http://docs.python.org/library/logging.html#formatter - - """ - - def format(self, record): - """Uses contextstring if request_id is set, otherwise default.""" - # NOTE(sdague): default the fancier formating params - # to an empty string so we don't throw an exception if - # they get used - for key in ('instance', 'color'): - if key not in record.__dict__: - record.__dict__[key] = '' - - if record.__dict__.get('request_id', None): - self._fmt = CONF.logging_context_format_string - else: - self._fmt = CONF.logging_default_format_string - - if (record.levelno == logging.DEBUG and - CONF.logging_debug_format_suffix): - self._fmt += " " + CONF.logging_debug_format_suffix - - # Cache this on the record, Logger will respect our formated copy - if record.exc_info: - record.exc_text = self.formatException(record.exc_info, record) - return logging.Formatter.format(self, record) - - def formatException(self, exc_info, record=None): - """Format exception output with CONF.logging_exception_prefix.""" - if not record: - return logging.Formatter.formatException(self, exc_info) - - stringbuffer = cStringIO.StringIO() - traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], - None, stringbuffer) - lines = stringbuffer.getvalue().split('\n') - stringbuffer.close() - - if CONF.logging_exception_prefix.find('%(asctime)') != -1: - record.asctime = self.formatTime(record, self.datefmt) - - formatted_lines = [] - for line in lines: - pl = CONF.logging_exception_prefix % record.__dict__ - fl = '%s%s' % (pl, line) - formatted_lines.append(fl) - return '\n'.join(formatted_lines) - - -class ColorHandler(logging.StreamHandler): - LEVEL_COLORS = { - logging.DEBUG: '\033[00;32m', # GREEN - logging.INFO: '\033[00;36m', # CYAN - logging.AUDIT: '\033[01;36m', # BOLD CYAN - logging.WARN: '\033[01;33m', # BOLD YELLOW - logging.ERROR: '\033[01;31m', # BOLD RED - logging.CRITICAL: '\033[01;31m', # BOLD RED - } - - def format(self, record): - record.color = self.LEVEL_COLORS[record.levelno] - return logging.StreamHandler.format(self, record) - - -class DeprecatedConfig(Exception): - message = _("Fatal call to deprecated config: %(msg)s") - - def __init__(self, msg): - super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/bufunfa/openstack/common/loopingcall.py b/bufunfa/openstack/common/loopingcall.py deleted file mode 100644 index 9246d5c..0000000 --- a/bufunfa/openstack/common/loopingcall.py +++ /dev/null @@ -1,88 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sys - -from eventlet import event -from eventlet import greenthread - -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common.gettextutils import _ - -LOG = logging.getLogger(__name__) - - -class LoopingCallDone(Exception): - """Exception to break out and stop a LoopingCall. - - The poll-function passed to LoopingCall can raise this exception to - break out of the loop normally. This is somewhat analogous to - StopIteration. - - An optional return-value can be included as the argument to the exception; - this return-value will be returned by LoopingCall.wait() - - """ - - def __init__(self, retvalue=True): - """:param retvalue: Value that LoopingCall.wait() should return.""" - self.retvalue = retvalue - - -class LoopingCall(object): - def __init__(self, f=None, *args, **kw): - self.args = args - self.kw = kw - self.f = f - self._running = False - - def start(self, interval, initial_delay=None): - self._running = True - done = event.Event() - - def _inner(): - if initial_delay: - greenthread.sleep(initial_delay) - - try: - while self._running: - self.f(*self.args, **self.kw) - if not self._running: - break - greenthread.sleep(interval) - except LoopingCallDone, e: - self.stop() - done.send(e.retvalue) - except Exception: - LOG.exception(_('in looping call')) - done.send_exception(*sys.exc_info()) - return - else: - done.send(True) - - self.done = done - - greenthread.spawn(_inner) - return self.done - - def stop(self): - self._running = False - - def wait(self): - return self.done.wait() diff --git a/bufunfa/openstack/common/network_utils.py b/bufunfa/openstack/common/network_utils.py deleted file mode 100644 index 69f6732..0000000 --- a/bufunfa/openstack/common/network_utils.py +++ /dev/null @@ -1,68 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Network-related utilities and helper functions. -""" - -import logging - -LOG = logging.getLogger(__name__) - - -def parse_host_port(address, default_port=None): - """ - Interpret a string as a host:port pair. - An IPv6 address MUST be escaped if accompanied by a port, - because otherwise ambiguity ensues: 2001:db8:85a3::8a2e:370:7334 - means both [2001:db8:85a3::8a2e:370:7334] and - [2001:db8:85a3::8a2e:370]:7334. - - >>> parse_host_port('server01:80') - ('server01', 80) - >>> parse_host_port('server01') - ('server01', None) - >>> parse_host_port('server01', default_port=1234) - ('server01', 1234) - >>> parse_host_port('[::1]:80') - ('::1', 80) - >>> parse_host_port('[::1]') - ('::1', None) - >>> parse_host_port('[::1]', default_port=1234) - ('::1', 1234) - >>> parse_host_port('2001:db8:85a3::8a2e:370:7334', default_port=1234) - ('2001:db8:85a3::8a2e:370:7334', 1234) - - """ - if address[0] == '[': - # Escaped ipv6 - _host, _port = address[1:].split(']') - host = _host - if ':' in _port: - port = _port.split(':')[1] - else: - port = default_port - else: - if address.count(':') == 1: - host, port = address.split(':') - else: - # 0 means ipv4, >1 means ipv6. - # We prohibit unescaped ipv6 addresses with port. - host = address - port = default_port - - return (host, None if port is None else int(port)) diff --git a/bufunfa/openstack/common/notifier/__init__.py b/bufunfa/openstack/common/notifier/__init__.py deleted file mode 100644 index 482d54e..0000000 --- a/bufunfa/openstack/common/notifier/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/bufunfa/openstack/common/notifier/api.py b/bufunfa/openstack/common/notifier/api.py deleted file mode 100644 index 3898cec..0000000 --- a/bufunfa/openstack/common/notifier/api.py +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import uuid - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import context -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import importutils -from bufunfa.openstack.common import jsonutils -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common import timeutils - - -LOG = logging.getLogger(__name__) - -notifier_opts = [ - cfg.MultiStrOpt('notification_driver', - default=[], - deprecated_name='list_notifier_drivers', - help='Driver or drivers to handle sending notifications'), - cfg.StrOpt('default_notification_level', - default='INFO', - help='Default notification level for outgoing notifications'), - cfg.StrOpt('default_publisher_id', - default='$host', - help='Default publisher_id for outgoing notifications'), -] - -CONF = cfg.CONF -CONF.register_opts(notifier_opts) - -WARN = 'WARN' -INFO = 'INFO' -ERROR = 'ERROR' -CRITICAL = 'CRITICAL' -DEBUG = 'DEBUG' - -log_levels = (DEBUG, WARN, INFO, ERROR, CRITICAL) - - -class BadPriorityException(Exception): - pass - - -def notify_decorator(name, fn): - """ decorator for notify which is used from utils.monkey_patch() - - :param name: name of the function - :param function: - object of the function - :returns: function -- decorated function - - """ - def wrapped_func(*args, **kwarg): - body = {} - body['args'] = [] - body['kwarg'] = {} - for arg in args: - body['args'].append(arg) - for key in kwarg: - body['kwarg'][key] = kwarg[key] - - ctxt = context.get_context_from_function_and_args(fn, args, kwarg) - notify(ctxt, - CONF.default_publisher_id, - name, - CONF.default_notification_level, - body) - return fn(*args, **kwarg) - return wrapped_func - - -def publisher_id(service, host=None): - if not host: - host = CONF.host - return "%s.%s" % (service, host) - - -def notify(context, publisher_id, event_type, priority, payload): - """Sends a notification using the specified driver - - :param publisher_id: the source worker_type.host of the message - :param event_type: the literal type of event (ex. Instance Creation) - :param priority: patterned after the enumeration of Python logging - levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL) - :param payload: A python dictionary of attributes - - Outgoing message format includes the above parameters, and appends the - following: - - message_id - a UUID representing the id for this notification - - timestamp - the GMT timestamp the notification was sent at - - The composite message will be constructed as a dictionary of the above - attributes, which will then be sent via the transport mechanism defined - by the driver. - - Message example:: - - {'message_id': str(uuid.uuid4()), - 'publisher_id': 'compute.host1', - 'timestamp': timeutils.utcnow(), - 'priority': 'WARN', - 'event_type': 'compute.create_instance', - 'payload': {'instance_id': 12, ... }} - - """ - if priority not in log_levels: - raise BadPriorityException( - _('%s not in valid priorities') % priority) - - # Ensure everything is JSON serializable. - payload = jsonutils.to_primitive(payload, convert_instances=True) - - msg = dict(message_id=str(uuid.uuid4()), - publisher_id=publisher_id, - event_type=event_type, - priority=priority, - payload=payload, - timestamp=str(timeutils.utcnow())) - - for driver in _get_drivers(): - try: - driver.notify(context, msg) - except Exception, e: - LOG.exception(_("Problem '%(e)s' attempting to " - "send to notification system. " - "Payload=%(payload)s") % locals()) - - -_drivers = None - - -def _get_drivers(): - """Instantiate, cache, and return drivers based on the CONF.""" - global _drivers - if _drivers is None: - _drivers = {} - for notification_driver in CONF.notification_driver: - add_driver(notification_driver) - - return _drivers.values() - - -def add_driver(notification_driver): - """Add a notification driver at runtime.""" - # Make sure the driver list is initialized. - _get_drivers() - if isinstance(notification_driver, basestring): - # Load and add - try: - driver = importutils.import_module(notification_driver) - _drivers[notification_driver] = driver - except ImportError as e: - LOG.exception(_("Failed to load notifier %s. " - "These notifications will not be sent.") % - notification_driver) - else: - # Driver is already loaded; just add the object. - _drivers[notification_driver] = notification_driver - - -def _reset_drivers(): - """Used by unit tests to reset the drivers.""" - global _drivers - _drivers = None diff --git a/bufunfa/openstack/common/notifier/list_notifier.py b/bufunfa/openstack/common/notifier/list_notifier.py deleted file mode 100644 index 320ac29..0000000 --- a/bufunfa/openstack/common/notifier/list_notifier.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import importutils -from bufunfa.openstack.common import log as logging - - -list_notifier_drivers_opt = cfg.MultiStrOpt( - 'list_notifier_drivers', - default=['bufunfa.openstack.common.notifier.no_op_notifier'], - help='List of drivers to send notifications') - -CONF = cfg.CONF -CONF.register_opt(list_notifier_drivers_opt) - -LOG = logging.getLogger(__name__) - -drivers = None - - -class ImportFailureNotifier(object): - """Noisily re-raises some exception over-and-over when notify is called.""" - - def __init__(self, exception): - self.exception = exception - - def notify(self, context, message): - raise self.exception - - -def _get_drivers(): - """Instantiates and returns drivers based on the flag values.""" - global drivers - if drivers is None: - drivers = [] - for notification_driver in CONF.list_notifier_drivers: - try: - drivers.append(importutils.import_module(notification_driver)) - except ImportError as e: - drivers.append(ImportFailureNotifier(e)) - return drivers - - -def add_driver(notification_driver): - """Add a notification driver at runtime.""" - # Make sure the driver list is initialized. - _get_drivers() - if isinstance(notification_driver, basestring): - # Load and add - try: - drivers.append(importutils.import_module(notification_driver)) - except ImportError as e: - drivers.append(ImportFailureNotifier(e)) - else: - # Driver is already loaded; just add the object. - drivers.append(notification_driver) - - -def _object_name(obj): - name = [] - if hasattr(obj, '__module__'): - name.append(obj.__module__) - if hasattr(obj, '__name__'): - name.append(obj.__name__) - else: - name.append(obj.__class__.__name__) - return '.'.join(name) - - -def remove_driver(notification_driver): - """Remove a notification driver at runtime.""" - # Make sure the driver list is initialized. - _get_drivers() - removed = False - if notification_driver in drivers: - # We're removing an object. Easy. - drivers.remove(notification_driver) - removed = True - else: - # We're removing a driver by name. Search for it. - for driver in drivers: - if _object_name(driver) == notification_driver: - drivers.remove(driver) - removed = True - - if not removed: - raise ValueError("Cannot remove; %s is not in list" % - notification_driver) - - -def notify(context, message): - """Passes notification to multiple notifiers in a list.""" - for driver in _get_drivers(): - try: - driver.notify(context, message) - except Exception as e: - LOG.exception(_("Problem '%(e)s' attempting to send to " - "notification driver %(driver)s."), locals()) - - -def _reset_drivers(): - """Used by unit tests to reset the drivers.""" - global drivers - drivers = None diff --git a/bufunfa/openstack/common/notifier/log_notifier.py b/bufunfa/openstack/common/notifier/log_notifier.py deleted file mode 100644 index feb64ff..0000000 --- a/bufunfa/openstack/common/notifier/log_notifier.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import jsonutils -from bufunfa.openstack.common import log as logging - - -CONF = cfg.CONF - - -def notify(_context, message): - """Notifies the recipient of the desired event given the model. - Log notifications using openstack's default logging system""" - - priority = message.get('priority', - CONF.default_notification_level) - priority = priority.lower() - logger = logging.getLogger( - 'bufunfa.openstack.common.notification.%s' % - message['event_type']) - getattr(logger, priority)(jsonutils.dumps(message)) diff --git a/bufunfa/openstack/common/notifier/no_op_notifier.py b/bufunfa/openstack/common/notifier/no_op_notifier.py deleted file mode 100644 index ee1ddbd..0000000 --- a/bufunfa/openstack/common/notifier/no_op_notifier.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -def notify(_context, message): - """Notifies the recipient of the desired event given the model""" - pass diff --git a/bufunfa/openstack/common/notifier/rabbit_notifier.py b/bufunfa/openstack/common/notifier/rabbit_notifier.py deleted file mode 100644 index 649dafe..0000000 --- a/bufunfa/openstack/common/notifier/rabbit_notifier.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import context as req_context -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common import rpc - -LOG = logging.getLogger(__name__) - -notification_topic_opt = cfg.ListOpt( - 'notification_topics', default=['notifications', ], - help='AMQP topic used for openstack notifications') - -CONF = cfg.CONF -CONF.register_opt(notification_topic_opt) - - -def notify(context, message): - """Sends a notification to the RabbitMQ""" - if not context: - context = req_context.get_admin_context() - priority = message.get('priority', - CONF.default_notification_level) - priority = priority.lower() - for topic in CONF.notification_topics: - topic = '%s.%s' % (topic, priority) - try: - rpc.notify(context, topic, message) - except Exception, e: - LOG.exception(_("Could not send notification to %(topic)s. " - "Payload=%(message)s"), locals()) diff --git a/bufunfa/openstack/common/notifier/test_notifier.py b/bufunfa/openstack/common/notifier/test_notifier.py deleted file mode 100644 index 5e34880..0000000 --- a/bufunfa/openstack/common/notifier/test_notifier.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -NOTIFICATIONS = [] - - -def notify(_context, message): - """Test notifier, stores notifications in memory for unittests.""" - NOTIFICATIONS.append(message) diff --git a/bufunfa/openstack/common/periodic_task.py b/bufunfa/openstack/common/periodic_task.py deleted file mode 100644 index 5e0a536..0000000 --- a/bufunfa/openstack/common/periodic_task.py +++ /dev/null @@ -1,112 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common.gettextutils import _ - -LOG = logging.getLogger(__name__) - - -def periodic_task(*args, **kwargs): - """Decorator to indicate that a method is a periodic task. - - This decorator can be used in two ways: - - 1. Without arguments '@periodic_task', this will be run on every tick - of the periodic scheduler. - - 2. With arguments, @periodic_task(ticks_between_runs=N), this will be - run on every N ticks of the periodic scheduler. - """ - def decorator(f): - f._periodic_task = True - f._ticks_between_runs = kwargs.pop('ticks_between_runs', 0) - return f - - # NOTE(sirp): The `if` is necessary to allow the decorator to be used with - # and without parens. - # - # In the 'with-parens' case (with kwargs present), this function needs to - # return a decorator function since the interpreter will invoke it like: - # - # periodic_task(*args, **kwargs)(f) - # - # In the 'without-parens' case, the original function will be passed - # in as the first argument, like: - # - # periodic_task(f) - if kwargs: - return decorator - else: - return decorator(args[0]) - - -class _PeriodicTasksMeta(type): - def __init__(cls, names, bases, dict_): - """Metaclass that allows us to collect decorated periodic tasks.""" - super(_PeriodicTasksMeta, cls).__init__(names, bases, dict_) - - # NOTE(sirp): if the attribute is not present then we must be the base - # class, so, go ahead and initialize it. If the attribute is present, - # then we're a subclass so make a copy of it so we don't step on our - # parent's toes. - try: - cls._periodic_tasks = cls._periodic_tasks[:] - except AttributeError: - cls._periodic_tasks = [] - - try: - cls._ticks_to_skip = cls._ticks_to_skip.copy() - except AttributeError: - cls._ticks_to_skip = {} - - # This uses __dict__ instead of - # inspect.getmembers(cls, inspect.ismethod) so only the methods of the - # current class are added when this class is scanned, and base classes - # are not added redundantly. - for value in cls.__dict__.values(): - if getattr(value, '_periodic_task', False): - task = value - name = task.__name__ - cls._periodic_tasks.append((name, task)) - cls._ticks_to_skip[name] = task._ticks_between_runs - - -class PeriodicTasks(object): - __metaclass__ = _PeriodicTasksMeta - - def run_periodic_tasks(self, *args, **kwargs): - """Tasks to be run at a periodic interval.""" - raise_on_error = kwargs.get('raise_on_error', False) - for task_name, task in self._periodic_tasks: - full_task_name = '.'.join([self.__class__.__name__, task_name]) - - ticks_to_skip = self._ticks_to_skip[task_name] - if ticks_to_skip > 0: - LOG.debug(_("Skipping %(full_task_name)s, %(ticks_to_skip)s" - " ticks left until next run"), locals()) - self._ticks_to_skip[task_name] -= 1 - continue - - self._ticks_to_skip[task_name] = task._ticks_between_runs - LOG.debug(_("Running periodic task %(full_task_name)s"), locals()) - - try: - task(self, *args, **kwargs) - except Exception as e: - if raise_on_error: - raise - LOG.exception(_("Error during %(full_task_name)s: %(e)s"), - locals()) diff --git a/bufunfa/openstack/common/rpc/__init__.py b/bufunfa/openstack/common/rpc/__init__.py deleted file mode 100644 index 758bd08..0000000 --- a/bufunfa/openstack/common/rpc/__init__.py +++ /dev/null @@ -1,270 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# Copyright 2011 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -A remote procedure call (rpc) abstraction. - -For some wrappers that add message versioning to rpc, see: - rpc.dispatcher - rpc.proxy -""" - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import importutils - - -rpc_opts = [ - cfg.StrOpt('rpc_backend', - default='%s.impl_kombu' % __package__, - help="The messaging module to use, defaults to kombu."), - cfg.IntOpt('rpc_thread_pool_size', - default=64, - help='Size of RPC thread pool'), - cfg.IntOpt('rpc_conn_pool_size', - default=30, - help='Size of RPC connection pool'), - cfg.IntOpt('rpc_response_timeout', - default=60, - help='Seconds to wait for a response from call or multicall'), - cfg.IntOpt('rpc_cast_timeout', - default=30, - help='Seconds to wait before a cast expires (TTL). ' - 'Only supported by impl_zmq.'), - cfg.ListOpt('allowed_rpc_exception_modules', - default=['bufunfa.openstack.common.exception', - 'nova.exception', - 'cinder.exception', - ], - help='Modules of exceptions that are permitted to be recreated' - 'upon receiving exception data from an rpc call.'), - cfg.BoolOpt('fake_rabbit', - default=False, - help='If passed, use a fake RabbitMQ provider'), - # - # The following options are not registered here, but are expected to be - # present. The project using this library must register these options with - # the configuration so that project-specific defaults may be defined. - # - #cfg.StrOpt('control_exchange', - # default='nova', - # help='AMQP exchange to connect to if using RabbitMQ or Qpid'), -] - -cfg.CONF.register_opts(rpc_opts) - - -def create_connection(new=True): - """Create a connection to the message bus used for rpc. - - For some example usage of creating a connection and some consumers on that - connection, see nova.service. - - :param new: Whether or not to create a new connection. A new connection - will be created by default. If new is False, the - implementation is free to return an existing connection from a - pool. - - :returns: An instance of openstack.common.rpc.common.Connection - """ - return _get_impl().create_connection(cfg.CONF, new=new) - - -def call(context, topic, msg, timeout=None): - """Invoke a remote method that returns something. - - :param context: Information that identifies the user that has made this - request. - :param topic: The topic to send the rpc message to. This correlates to the - topic argument of - openstack.common.rpc.common.Connection.create_consumer() - and only applies when the consumer was created with - fanout=False. - :param msg: This is a dict in the form { "method" : "method_to_invoke", - "args" : dict_of_kwargs } - :param timeout: int, number of seconds to use for a response timeout. - If set, this overrides the rpc_response_timeout option. - - :returns: A dict from the remote method. - - :raises: openstack.common.rpc.common.Timeout if a complete response - is not received before the timeout is reached. - """ - return _get_impl().call(cfg.CONF, context, topic, msg, timeout) - - -def cast(context, topic, msg): - """Invoke a remote method that does not return anything. - - :param context: Information that identifies the user that has made this - request. - :param topic: The topic to send the rpc message to. This correlates to the - topic argument of - openstack.common.rpc.common.Connection.create_consumer() - and only applies when the consumer was created with - fanout=False. - :param msg: This is a dict in the form { "method" : "method_to_invoke", - "args" : dict_of_kwargs } - - :returns: None - """ - return _get_impl().cast(cfg.CONF, context, topic, msg) - - -def fanout_cast(context, topic, msg): - """Broadcast a remote method invocation with no return. - - This method will get invoked on all consumers that were set up with this - topic name and fanout=True. - - :param context: Information that identifies the user that has made this - request. - :param topic: The topic to send the rpc message to. This correlates to the - topic argument of - openstack.common.rpc.common.Connection.create_consumer() - and only applies when the consumer was created with - fanout=True. - :param msg: This is a dict in the form { "method" : "method_to_invoke", - "args" : dict_of_kwargs } - - :returns: None - """ - return _get_impl().fanout_cast(cfg.CONF, context, topic, msg) - - -def multicall(context, topic, msg, timeout=None): - """Invoke a remote method and get back an iterator. - - In this case, the remote method will be returning multiple values in - separate messages, so the return values can be processed as the come in via - an iterator. - - :param context: Information that identifies the user that has made this - request. - :param topic: The topic to send the rpc message to. This correlates to the - topic argument of - openstack.common.rpc.common.Connection.create_consumer() - and only applies when the consumer was created with - fanout=False. - :param msg: This is a dict in the form { "method" : "method_to_invoke", - "args" : dict_of_kwargs } - :param timeout: int, number of seconds to use for a response timeout. - If set, this overrides the rpc_response_timeout option. - - :returns: An iterator. The iterator will yield a tuple (N, X) where N is - an index that starts at 0 and increases by one for each value - returned and X is the Nth value that was returned by the remote - method. - - :raises: openstack.common.rpc.common.Timeout if a complete response - is not received before the timeout is reached. - """ - return _get_impl().multicall(cfg.CONF, context, topic, msg, timeout) - - -def notify(context, topic, msg): - """Send notification event. - - :param context: Information that identifies the user that has made this - request. - :param topic: The topic to send the notification to. - :param msg: This is a dict of content of event. - - :returns: None - """ - return _get_impl().notify(cfg.CONF, context, topic, msg) - - -def cleanup(): - """Clean up resoruces in use by implementation. - - Clean up any resources that have been allocated by the RPC implementation. - This is typically open connections to a messaging service. This function - would get called before an application using this API exits to allow - connections to get torn down cleanly. - - :returns: None - """ - return _get_impl().cleanup() - - -def cast_to_server(context, server_params, topic, msg): - """Invoke a remote method that does not return anything. - - :param context: Information that identifies the user that has made this - request. - :param server_params: Connection information - :param topic: The topic to send the notification to. - :param msg: This is a dict in the form { "method" : "method_to_invoke", - "args" : dict_of_kwargs } - - :returns: None - """ - return _get_impl().cast_to_server(cfg.CONF, context, server_params, topic, - msg) - - -def fanout_cast_to_server(context, server_params, topic, msg): - """Broadcast to a remote method invocation with no return. - - :param context: Information that identifies the user that has made this - request. - :param server_params: Connection information - :param topic: The topic to send the notification to. - :param msg: This is a dict in the form { "method" : "method_to_invoke", - "args" : dict_of_kwargs } - - :returns: None - """ - return _get_impl().fanout_cast_to_server(cfg.CONF, context, server_params, - topic, msg) - - -def queue_get_for(context, topic, host): - """Get a queue name for a given topic + host. - - This function only works if this naming convention is followed on the - consumer side, as well. For example, in nova, every instance of the - nova-foo service calls create_consumer() for two topics: - - foo - foo. - - Messages sent to the 'foo' topic are distributed to exactly one instance of - the nova-foo service. The services are chosen in a round-robin fashion. - Messages sent to the 'foo.' topic are sent to the nova-foo service on - . - """ - return '%s.%s' % (topic, host) - - -_RPCIMPL = None - - -def _get_impl(): - """Delay import of rpc_backend until configuration is loaded.""" - global _RPCIMPL - if _RPCIMPL is None: - try: - _RPCIMPL = importutils.import_module(cfg.CONF.rpc_backend) - except ImportError: - # For backwards compatibility with older nova config. - impl = cfg.CONF.rpc_backend.replace('nova.rpc', - 'nova.openstack.common.rpc') - _RPCIMPL = importutils.import_module(impl) - return _RPCIMPL diff --git a/bufunfa/openstack/common/rpc/amqp.py b/bufunfa/openstack/common/rpc/amqp.py deleted file mode 100644 index e8079e6..0000000 --- a/bufunfa/openstack/common/rpc/amqp.py +++ /dev/null @@ -1,429 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# Copyright 2011 - 2012, Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Shared code between AMQP based openstack.common.rpc implementations. - -The code in this module is shared between the rpc implemenations based on AMQP. -Specifically, this includes impl_kombu and impl_qpid. impl_carrot also uses -AMQP, but is deprecated and predates this code. -""" - -import inspect -import logging -import sys -import uuid - -from eventlet import greenpool -from eventlet import pools -from eventlet import semaphore - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import excutils -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import local -from bufunfa.openstack.common.rpc import common as rpc_common - - -LOG = logging.getLogger(__name__) - - -class Pool(pools.Pool): - """Class that implements a Pool of Connections.""" - def __init__(self, conf, connection_cls, *args, **kwargs): - self.connection_cls = connection_cls - self.conf = conf - kwargs.setdefault("max_size", self.conf.rpc_conn_pool_size) - kwargs.setdefault("order_as_stack", True) - super(Pool, self).__init__(*args, **kwargs) - - # TODO(comstud): Timeout connections not used in a while - def create(self): - LOG.debug('Pool creating new connection') - return self.connection_cls(self.conf) - - def empty(self): - while self.free_items: - self.get().close() - - -_pool_create_sem = semaphore.Semaphore() - - -def get_connection_pool(conf, connection_cls): - with _pool_create_sem: - # Make sure only one thread tries to create the connection pool. - if not connection_cls.pool: - connection_cls.pool = Pool(conf, connection_cls) - return connection_cls.pool - - -class ConnectionContext(rpc_common.Connection): - """The class that is actually returned to the caller of - create_connection(). This is essentially a wrapper around - Connection that supports 'with'. It can also return a new - Connection, or one from a pool. The function will also catch - when an instance of this class is to be deleted. With that - we can return Connections to the pool on exceptions and so - forth without making the caller be responsible for catching - them. If possible the function makes sure to return a - connection to the pool. - """ - - def __init__(self, conf, connection_pool, pooled=True, server_params=None): - """Create a new connection, or get one from the pool""" - self.connection = None - self.conf = conf - self.connection_pool = connection_pool - if pooled: - self.connection = connection_pool.get() - else: - self.connection = connection_pool.connection_cls( - conf, - server_params=server_params) - self.pooled = pooled - - def __enter__(self): - """When with ConnectionContext() is used, return self""" - return self - - def _done(self): - """If the connection came from a pool, clean it up and put it back. - If it did not come from a pool, close it. - """ - if self.connection: - if self.pooled: - # Reset the connection so it's ready for the next caller - # to grab from the pool - self.connection.reset() - self.connection_pool.put(self.connection) - else: - try: - self.connection.close() - except Exception: - pass - self.connection = None - - def __exit__(self, exc_type, exc_value, tb): - """End of 'with' statement. We're done here.""" - self._done() - - def __del__(self): - """Caller is done with this connection. Make sure we cleaned up.""" - self._done() - - def close(self): - """Caller is done with this connection.""" - self._done() - - def create_consumer(self, topic, proxy, fanout=False): - self.connection.create_consumer(topic, proxy, fanout) - - def create_worker(self, topic, proxy, pool_name): - self.connection.create_worker(topic, proxy, pool_name) - - def consume_in_thread(self): - self.connection.consume_in_thread() - - def consume_in_thread_group(self, thread_group): - self.connection.consume_in_thread_group(thread_group) - - def __getattr__(self, key): - """Proxy all other calls to the Connection instance""" - if self.connection: - return getattr(self.connection, key) - else: - raise rpc_common.InvalidRPCConnectionReuse() - - -def msg_reply(conf, msg_id, connection_pool, reply=None, failure=None, - ending=False): - """Sends a reply or an error on the channel signified by msg_id. - - Failure should be a sys.exc_info() tuple. - - """ - with ConnectionContext(conf, connection_pool) as conn: - if failure: - failure = rpc_common.serialize_remote_exception(failure) - - try: - msg = {'result': reply, 'failure': failure} - except TypeError: - msg = {'result': dict((k, repr(v)) - for k, v in reply.__dict__.iteritems()), - 'failure': failure} - if ending: - msg['ending'] = True - conn.direct_send(msg_id, msg) - - -class RpcContext(rpc_common.CommonRpcContext): - """Context that supports replying to a rpc.call""" - def __init__(self, **kwargs): - self.msg_id = kwargs.pop('msg_id', None) - self.conf = kwargs.pop('conf') - super(RpcContext, self).__init__(**kwargs) - - def deepcopy(self): - values = self.to_dict() - values['conf'] = self.conf - values['msg_id'] = self.msg_id - return self.__class__(**values) - - def reply(self, reply=None, failure=None, ending=False, - connection_pool=None): - if self.msg_id: - msg_reply(self.conf, self.msg_id, connection_pool, reply, failure, - ending) - if ending: - self.msg_id = None - - -def unpack_context(conf, msg): - """Unpack context from msg.""" - context_dict = {} - for key in list(msg.keys()): - # NOTE(vish): Some versions of python don't like unicode keys - # in kwargs. - key = str(key) - if key.startswith('_context_'): - value = msg.pop(key) - context_dict[key[9:]] = value - context_dict['msg_id'] = msg.pop('_msg_id', None) - context_dict['conf'] = conf - ctx = RpcContext.from_dict(context_dict) - rpc_common._safe_log(LOG.debug, _('unpacked context: %s'), ctx.to_dict()) - return ctx - - -def pack_context(msg, context): - """Pack context into msg. - - Values for message keys need to be less than 255 chars, so we pull - context out into a bunch of separate keys. If we want to support - more arguments in rabbit messages, we may want to do the same - for args at some point. - - """ - context_d = dict([('_context_%s' % key, value) - for (key, value) in context.to_dict().iteritems()]) - msg.update(context_d) - - -class ProxyCallback(object): - """Calls methods on a proxy object based on method and args.""" - - def __init__(self, conf, proxy, connection_pool): - self.proxy = proxy - self.pool = greenpool.GreenPool(conf.rpc_thread_pool_size) - self.connection_pool = connection_pool - self.conf = conf - - def __call__(self, message_data): - """Consumer callback to call a method on a proxy object. - - Parses the message for validity and fires off a thread to call the - proxy object method. - - Message data should be a dictionary with two keys: - method: string representing the method to call - args: dictionary of arg: value - - Example: {'method': 'echo', 'args': {'value': 42}} - - """ - # It is important to clear the context here, because at this point - # the previous context is stored in local.store.context - if hasattr(local.store, 'context'): - del local.store.context - rpc_common._safe_log(LOG.debug, _('received %s'), message_data) - ctxt = unpack_context(self.conf, message_data) - method = message_data.get('method') - args = message_data.get('args', {}) - version = message_data.get('version', None) - if not method: - LOG.warn(_('no method for message: %s') % message_data) - ctxt.reply(_('No method for message: %s') % message_data, - connection_pool=self.connection_pool) - return - self.pool.spawn_n(self._process_data, ctxt, version, method, args) - - def _process_data(self, ctxt, version, method, args): - """Process a message in a new thread. - - If the proxy object we have has a dispatch method - (see rpc.dispatcher.RpcDispatcher), pass it the version, - method, and args and let it dispatch as appropriate. If not, use - the old behavior of magically calling the specified method on the - proxy we have here. - """ - ctxt.update_store() - try: - rval = self.proxy.dispatch(ctxt, version, method, **args) - # Check if the result was a generator - if inspect.isgenerator(rval): - for x in rval: - ctxt.reply(x, None, connection_pool=self.connection_pool) - else: - ctxt.reply(rval, None, connection_pool=self.connection_pool) - # This final None tells multicall that it is done. - ctxt.reply(ending=True, connection_pool=self.connection_pool) - except Exception as e: - LOG.exception('Exception during message handling') - ctxt.reply(None, sys.exc_info(), - connection_pool=self.connection_pool) - - -class MulticallWaiter(object): - def __init__(self, conf, connection, timeout): - self._connection = connection - self._iterator = connection.iterconsume(timeout=timeout or - conf.rpc_response_timeout) - self._result = None - self._done = False - self._got_ending = False - self._conf = conf - - def done(self): - if self._done: - return - self._done = True - self._iterator.close() - self._iterator = None - self._connection.close() - - def __call__(self, data): - """The consume() callback will call this. Store the result.""" - if data['failure']: - failure = data['failure'] - self._result = rpc_common.deserialize_remote_exception(self._conf, - failure) - - elif data.get('ending', False): - self._got_ending = True - else: - self._result = data['result'] - - def __iter__(self): - """Return a result until we get a 'None' response from consumer""" - if self._done: - raise StopIteration - while True: - try: - self._iterator.next() - except Exception: - with excutils.save_and_reraise_exception(): - self.done() - if self._got_ending: - self.done() - raise StopIteration - result = self._result - if isinstance(result, Exception): - self.done() - raise result - yield result - - -def create_connection(conf, new, connection_pool): - """Create a connection""" - return ConnectionContext(conf, connection_pool, pooled=not new) - - -def multicall(conf, context, topic, msg, timeout, connection_pool): - """Make a call that returns multiple times.""" - # Can't use 'with' for multicall, as it returns an iterator - # that will continue to use the connection. When it's done, - # connection.close() will get called which will put it back into - # the pool - LOG.debug(_('Making asynchronous call on %s ...'), topic) - msg_id = uuid.uuid4().hex - msg.update({'_msg_id': msg_id}) - LOG.debug(_('MSG_ID is %s') % (msg_id)) - pack_context(msg, context) - - conn = ConnectionContext(conf, connection_pool) - wait_msg = MulticallWaiter(conf, conn, timeout) - conn.declare_direct_consumer(msg_id, wait_msg) - conn.topic_send(topic, msg) - return wait_msg - - -def call(conf, context, topic, msg, timeout, connection_pool): - """Sends a message on a topic and wait for a response.""" - rv = multicall(conf, context, topic, msg, timeout, connection_pool) - # NOTE(vish): return the last result from the multicall - rv = list(rv) - if not rv: - return - return rv[-1] - - -def cast(conf, context, topic, msg, connection_pool): - """Sends a message on a topic without waiting for a response.""" - LOG.debug(_('Making asynchronous cast on %s...'), topic) - pack_context(msg, context) - with ConnectionContext(conf, connection_pool) as conn: - conn.topic_send(topic, msg) - - -def fanout_cast(conf, context, topic, msg, connection_pool): - """Sends a message on a fanout exchange without waiting for a response.""" - LOG.debug(_('Making asynchronous fanout cast...')) - pack_context(msg, context) - with ConnectionContext(conf, connection_pool) as conn: - conn.fanout_send(topic, msg) - - -def cast_to_server(conf, context, server_params, topic, msg, connection_pool): - """Sends a message on a topic to a specific server.""" - pack_context(msg, context) - with ConnectionContext(conf, connection_pool, pooled=False, - server_params=server_params) as conn: - conn.topic_send(topic, msg) - - -def fanout_cast_to_server(conf, context, server_params, topic, msg, - connection_pool): - """Sends a message on a fanout exchange to a specific server.""" - pack_context(msg, context) - with ConnectionContext(conf, connection_pool, pooled=False, - server_params=server_params) as conn: - conn.fanout_send(topic, msg) - - -def notify(conf, context, topic, msg, connection_pool): - """Sends a notification event on a topic.""" - event_type = msg.get('event_type') - LOG.debug(_('Sending %(event_type)s on %(topic)s'), locals()) - pack_context(msg, context) - with ConnectionContext(conf, connection_pool) as conn: - conn.notify_send(topic, msg) - - -def cleanup(connection_pool): - if connection_pool: - connection_pool.empty() - - -def get_control_exchange(conf): - try: - return conf.control_exchange - except cfg.NoSuchOptError: - return 'openstack' diff --git a/bufunfa/openstack/common/rpc/common.py b/bufunfa/openstack/common/rpc/common.py deleted file mode 100644 index 15c8bf0..0000000 --- a/bufunfa/openstack/common/rpc/common.py +++ /dev/null @@ -1,324 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# Copyright 2011 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import logging -import traceback - -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import importutils -from bufunfa.openstack.common import jsonutils -from bufunfa.openstack.common import local - - -LOG = logging.getLogger(__name__) - - -class RPCException(Exception): - message = _("An unknown RPC related exception occurred.") - - def __init__(self, message=None, **kwargs): - self.kwargs = kwargs - - if not message: - try: - message = self.message % kwargs - - except Exception as e: - # kwargs doesn't match a variable in the message - # log the issue and the kwargs - LOG.exception(_('Exception in string format operation')) - for name, value in kwargs.iteritems(): - LOG.error("%s: %s" % (name, value)) - # at least get the core message out if something happened - message = self.message - - super(RPCException, self).__init__(message) - - -class RemoteError(RPCException): - """Signifies that a remote class has raised an exception. - - Contains a string representation of the type of the original exception, - the value of the original exception, and the traceback. These are - sent to the parent as a joined string so printing the exception - contains all of the relevant info. - - """ - message = _("Remote error: %(exc_type)s %(value)s\n%(traceback)s.") - - def __init__(self, exc_type=None, value=None, traceback=None): - self.exc_type = exc_type - self.value = value - self.traceback = traceback - super(RemoteError, self).__init__(exc_type=exc_type, - value=value, - traceback=traceback) - - -class Timeout(RPCException): - """Signifies that a timeout has occurred. - - This exception is raised if the rpc_response_timeout is reached while - waiting for a response from the remote side. - """ - message = _("Timeout while waiting on RPC response.") - - -class InvalidRPCConnectionReuse(RPCException): - message = _("Invalid reuse of an RPC connection.") - - -class UnsupportedRpcVersion(RPCException): - message = _("Specified RPC version, %(version)s, not supported by " - "this endpoint.") - - -class Connection(object): - """A connection, returned by rpc.create_connection(). - - This class represents a connection to the message bus used for rpc. - An instance of this class should never be created by users of the rpc API. - Use rpc.create_connection() instead. - """ - def close(self): - """Close the connection. - - This method must be called when the connection will no longer be used. - It will ensure that any resources associated with the connection, such - as a network connection, and cleaned up. - """ - raise NotImplementedError() - - def create_consumer(self, topic, proxy, fanout=False): - """Create a consumer on this connection. - - A consumer is associated with a message queue on the backend message - bus. The consumer will read messages from the queue, unpack them, and - dispatch them to the proxy object. The contents of the message pulled - off of the queue will determine which method gets called on the proxy - object. - - :param topic: This is a name associated with what to consume from. - Multiple instances of a service may consume from the same - topic. For example, all instances of nova-compute consume - from a queue called "compute". In that case, the - messages will get distributed amongst the consumers in a - round-robin fashion if fanout=False. If fanout=True, - every consumer associated with this topic will get a - copy of every message. - :param proxy: The object that will handle all incoming messages. - :param fanout: Whether or not this is a fanout topic. See the - documentation for the topic parameter for some - additional comments on this. - """ - raise NotImplementedError() - - def create_worker(self, topic, proxy, pool_name): - """Create a worker on this connection. - - A worker is like a regular consumer of messages directed to a - topic, except that it is part of a set of such consumers (the - "pool") which may run in parallel. Every pool of workers will - receive a given message, but only one worker in the pool will - be asked to process it. Load is distributed across the members - of the pool in round-robin fashion. - - :param topic: This is a name associated with what to consume from. - Multiple instances of a service may consume from the same - topic. - :param proxy: The object that will handle all incoming messages. - :param pool_name: String containing the name of the pool of workers - """ - raise NotImplementedError() - - def consume_in_thread(self): - """Spawn a thread to handle incoming messages. - - Spawn a thread that will be responsible for handling all incoming - messages for consumers that were set up on this connection. - - Message dispatching inside of this is expected to be implemented in a - non-blocking manner. An example implementation would be having this - thread pull messages in for all of the consumers, but utilize a thread - pool for dispatching the messages to the proxy objects. - """ - raise NotImplementedError() - - def consume_in_thread_group(self, thread_group): - """Spawn a thread to handle incoming messages in the supplied ThreadGroup. - - Spawn a thread that will be responsible for handling all incoming - messages for consumers that were set up on this connection. - - Message dispatching inside of this is expected to be implemented in a - non-blocking manner. An example implementation would be having this - thread pull messages in for all of the consumers, but utilize a thread - pool for dispatching the messages to the proxy objects. - """ - raise NotImplementedError() - - -def _safe_log(log_func, msg, msg_data): - """Sanitizes the msg_data field before logging.""" - SANITIZE = {'set_admin_password': ('new_pass',), - 'run_instance': ('admin_password',), } - - has_method = 'method' in msg_data and msg_data['method'] in SANITIZE - has_context_token = '_context_auth_token' in msg_data - has_token = 'auth_token' in msg_data - - if not any([has_method, has_context_token, has_token]): - return log_func(msg, msg_data) - - msg_data = copy.deepcopy(msg_data) - - if has_method: - method = msg_data['method'] - if method in SANITIZE: - args_to_sanitize = SANITIZE[method] - for arg in args_to_sanitize: - try: - msg_data['args'][arg] = "" - except KeyError: - pass - - if has_context_token: - msg_data['_context_auth_token'] = '' - - if has_token: - msg_data['auth_token'] = '' - - return log_func(msg, msg_data) - - -def serialize_remote_exception(failure_info): - """Prepares exception data to be sent over rpc. - - Failure_info should be a sys.exc_info() tuple. - - """ - tb = traceback.format_exception(*failure_info) - failure = failure_info[1] - LOG.error(_("Returning exception %s to caller"), unicode(failure)) - LOG.error(tb) - - kwargs = {} - if hasattr(failure, 'kwargs'): - kwargs = failure.kwargs - - data = { - 'class': str(failure.__class__.__name__), - 'module': str(failure.__class__.__module__), - 'message': unicode(failure), - 'tb': tb, - 'args': failure.args, - 'kwargs': kwargs - } - - json_data = jsonutils.dumps(data) - - return json_data - - -def deserialize_remote_exception(conf, data): - failure = jsonutils.loads(str(data)) - - trace = failure.get('tb', []) - message = failure.get('message', "") + "\n" + "\n".join(trace) - name = failure.get('class') - module = failure.get('module') - - # NOTE(ameade): We DO NOT want to allow just any module to be imported, in - # order to prevent arbitrary code execution. - if not module in conf.allowed_rpc_exception_modules: - return RemoteError(name, failure.get('message'), trace) - - try: - mod = importutils.import_module(module) - klass = getattr(mod, name) - if not issubclass(klass, Exception): - raise TypeError("Can only deserialize Exceptions") - - failure = klass(**failure.get('kwargs', {})) - except (AttributeError, TypeError, ImportError): - return RemoteError(name, failure.get('message'), trace) - - ex_type = type(failure) - str_override = lambda self: message - new_ex_type = type(ex_type.__name__ + "_Remote", (ex_type,), - {'__str__': str_override, '__unicode__': str_override}) - try: - # NOTE(ameade): Dynamically create a new exception type and swap it in - # as the new type for the exception. This only works on user defined - # Exceptions and not core python exceptions. This is important because - # we cannot necessarily change an exception message so we must override - # the __str__ method. - failure.__class__ = new_ex_type - except TypeError as e: - # NOTE(ameade): If a core exception then just add the traceback to the - # first exception argument. - failure.args = (message,) + failure.args[1:] - return failure - - -class CommonRpcContext(object): - def __init__(self, **kwargs): - self.values = kwargs - - def __getattr__(self, key): - try: - return self.values[key] - except KeyError: - raise AttributeError(key) - - def to_dict(self): - return copy.deepcopy(self.values) - - @classmethod - def from_dict(cls, values): - return cls(**values) - - def deepcopy(self): - return self.from_dict(self.to_dict()) - - def update_store(self): - local.store.context = self - - def elevated(self, read_deleted=None, overwrite=False): - """Return a version of this context with admin flag set.""" - # TODO(russellb) This method is a bit of a nova-ism. It makes - # some assumptions about the data in the request context sent - # across rpc, while the rest of this class does not. We could get - # rid of this if we changed the nova code that uses this to - # convert the RpcContext back to its native RequestContext doing - # something like nova.context.RequestContext.from_dict(ctxt.to_dict()) - - context = self.deepcopy() - context.values['is_admin'] = True - - context.values.setdefault('roles', []) - - if 'admin' not in context.values['roles']: - context.values['roles'].append('admin') - - if read_deleted is not None: - context.values['read_deleted'] = read_deleted - - return context diff --git a/bufunfa/openstack/common/rpc/dispatcher.py b/bufunfa/openstack/common/rpc/dispatcher.py deleted file mode 100644 index dc81273..0000000 --- a/bufunfa/openstack/common/rpc/dispatcher.py +++ /dev/null @@ -1,150 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Code for rpc message dispatching. - -Messages that come in have a version number associated with them. RPC API -version numbers are in the form: - - Major.Minor - -For a given message with version X.Y, the receiver must be marked as able to -handle messages of version A.B, where: - - A = X - - B >= Y - -The Major version number would be incremented for an almost completely new API. -The Minor version number would be incremented for backwards compatible changes -to an existing API. A backwards compatible change could be something like -adding a new method, adding an argument to an existing method (but not -requiring it), or changing the type for an existing argument (but still -handling the old type as well). - -The conversion over to a versioned API must be done on both the client side and -server side of the API at the same time. However, as the code stands today, -there can be both versioned and unversioned APIs implemented in the same code -base. - - -EXAMPLES: - -Nova was the first project to use versioned rpc APIs. Consider the compute rpc -API as an example. The client side is in nova/compute/rpcapi.py and the server -side is in nova/compute/manager.py. - - -Example 1) Adding a new method. - -Adding a new method is a backwards compatible change. It should be added to -nova/compute/manager.py, and RPC_API_VERSION should be bumped from X.Y to -X.Y+1. On the client side, the new method in nova/compute/rpcapi.py should -have a specific version specified to indicate the minimum API version that must -be implemented for the method to be supported. For example: - - def get_host_uptime(self, ctxt, host): - topic = _compute_topic(self.topic, ctxt, host, None) - return self.call(ctxt, self.make_msg('get_host_uptime'), topic, - version='1.1') - -In this case, version '1.1' is the first version that supported the -get_host_uptime() method. - - -Example 2) Adding a new parameter. - -Adding a new parameter to an rpc method can be made backwards compatible. The -RPC_API_VERSION on the server side (nova/compute/manager.py) should be bumped. -The implementation of the method must not expect the parameter to be present. - - def some_remote_method(self, arg1, arg2, newarg=None): - # The code needs to deal with newarg=None for cases - # where an older client sends a message without it. - pass - -On the client side, the same changes should be made as in example 1. The -minimum version that supports the new parameter should be specified. -""" - -from bufunfa.openstack.common.rpc import common as rpc_common - - -class RpcDispatcher(object): - """Dispatch rpc messages according to the requested API version. - - This class can be used as the top level 'manager' for a service. It - contains a list of underlying managers that have an API_VERSION attribute. - """ - - def __init__(self, callbacks): - """Initialize the rpc dispatcher. - - :param callbacks: List of proxy objects that are an instance - of a class with rpc methods exposed. Each proxy - object should have an RPC_API_VERSION attribute. - """ - self.callbacks = callbacks - super(RpcDispatcher, self).__init__() - - @staticmethod - def _is_compatible(mversion, version): - """Determine whether versions are compatible. - - :param mversion: The API version implemented by a callback. - :param version: The API version requested by an incoming message. - """ - version_parts = version.split('.') - mversion_parts = mversion.split('.') - if int(version_parts[0]) != int(mversion_parts[0]): # Major - return False - if int(version_parts[1]) > int(mversion_parts[1]): # Minor - return False - return True - - def dispatch(self, ctxt, version, method, **kwargs): - """Dispatch a message based on a requested version. - - :param ctxt: The request context - :param version: The requested API version from the incoming message - :param method: The method requested to be called by the incoming - message. - :param kwargs: A dict of keyword arguments to be passed to the method. - - :returns: Whatever is returned by the underlying method that gets - called. - """ - if not version: - version = '1.0' - - had_compatible = False - for proxyobj in self.callbacks: - if hasattr(proxyobj, 'RPC_API_VERSION'): - rpc_api_version = proxyobj.RPC_API_VERSION - else: - rpc_api_version = '1.0' - is_compatible = self._is_compatible(rpc_api_version, version) - had_compatible = had_compatible or is_compatible - if not hasattr(proxyobj, method): - continue - if is_compatible: - return getattr(proxyobj, method)(ctxt, **kwargs) - - if had_compatible: - raise AttributeError("No such RPC function '%s'" % method) - else: - raise rpc_common.UnsupportedRpcVersion(version=version) diff --git a/bufunfa/openstack/common/rpc/impl_fake.py b/bufunfa/openstack/common/rpc/impl_fake.py deleted file mode 100644 index 72a4e0a..0000000 --- a/bufunfa/openstack/common/rpc/impl_fake.py +++ /dev/null @@ -1,187 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Fake RPC implementation which calls proxy methods directly with no -queues. Casts will block, but this is very useful for tests. -""" - -import inspect -import time - -import eventlet - -from bufunfa.openstack.common import jsonutils -from bufunfa.openstack.common.rpc import common as rpc_common - -CONSUMERS = {} - - -class RpcContext(rpc_common.CommonRpcContext): - def __init__(self, **kwargs): - super(RpcContext, self).__init__(**kwargs) - self._response = [] - self._done = False - - def deepcopy(self): - values = self.to_dict() - new_inst = self.__class__(**values) - new_inst._response = self._response - new_inst._done = self._done - return new_inst - - def reply(self, reply=None, failure=None, ending=False): - if ending: - self._done = True - if not self._done: - self._response.append((reply, failure)) - - -class Consumer(object): - def __init__(self, topic, proxy): - self.topic = topic - self.proxy = proxy - - def call(self, context, version, method, args, timeout): - done = eventlet.event.Event() - - def _inner(): - ctxt = RpcContext.from_dict(context.to_dict()) - try: - rval = self.proxy.dispatch(context, version, method, **args) - res = [] - # Caller might have called ctxt.reply() manually - for (reply, failure) in ctxt._response: - if failure: - raise failure[0], failure[1], failure[2] - res.append(reply) - # if ending not 'sent'...we might have more data to - # return from the function itself - if not ctxt._done: - if inspect.isgenerator(rval): - for val in rval: - res.append(val) - else: - res.append(rval) - done.send(res) - except Exception as e: - done.send_exception(e) - - thread = eventlet.greenthread.spawn(_inner) - - if timeout: - start_time = time.time() - while not done.ready(): - eventlet.greenthread.sleep(1) - cur_time = time.time() - if (cur_time - start_time) > timeout: - thread.kill() - raise rpc_common.Timeout() - - return done.wait() - - -class Connection(object): - """Connection object.""" - - def __init__(self): - self.consumers = [] - - def create_consumer(self, topic, proxy, fanout=False): - consumer = Consumer(topic, proxy) - self.consumers.append(consumer) - if topic not in CONSUMERS: - CONSUMERS[topic] = [] - CONSUMERS[topic].append(consumer) - - def close(self): - for consumer in self.consumers: - CONSUMERS[consumer.topic].remove(consumer) - self.consumers = [] - - def consume_in_thread(self): - pass - - def consume_in_thread_group(self, thread_group): - pass - - -def create_connection(conf, new=True): - """Create a connection""" - return Connection() - - -def check_serialize(msg): - """Make sure a message intended for rpc can be serialized.""" - jsonutils.dumps(msg) - - -def multicall(conf, context, topic, msg, timeout=None): - """Make a call that returns multiple times.""" - - check_serialize(msg) - - method = msg.get('method') - if not method: - return - args = msg.get('args', {}) - version = msg.get('version', None) - - try: - consumer = CONSUMERS[topic][0] - except (KeyError, IndexError): - return iter([None]) - else: - return consumer.call(context, version, method, args, timeout) - - -def call(conf, context, topic, msg, timeout=None): - """Sends a message on a topic and wait for a response.""" - rv = multicall(conf, context, topic, msg, timeout) - # NOTE(vish): return the last result from the multicall - rv = list(rv) - if not rv: - return - return rv[-1] - - -def cast(conf, context, topic, msg): - try: - call(conf, context, topic, msg) - except Exception: - pass - - -def notify(conf, context, topic, msg): - check_serialize(msg) - - -def cleanup(): - pass - - -def fanout_cast(conf, context, topic, msg): - """Cast to all consumers of a topic""" - check_serialize(msg) - method = msg.get('method') - if not method: - return - args = msg.get('args', {}) - version = msg.get('version', None) - - for consumer in CONSUMERS.get(topic, []): - try: - consumer.call(context, version, method, args, None) - except Exception: - pass diff --git a/bufunfa/openstack/common/rpc/impl_kombu.py b/bufunfa/openstack/common/rpc/impl_kombu.py deleted file mode 100644 index 7734769..0000000 --- a/bufunfa/openstack/common/rpc/impl_kombu.py +++ /dev/null @@ -1,801 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import functools -import itertools -import socket -import ssl -import sys -import time -import uuid - -import eventlet -import greenlet -import kombu -import kombu.connection -import kombu.entity -import kombu.messaging - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import network_utils -from bufunfa.openstack.common.rpc import amqp as rpc_amqp -from bufunfa.openstack.common.rpc import common as rpc_common - -kombu_opts = [ - cfg.StrOpt('kombu_ssl_version', - default='', - help='SSL version to use (valid only if SSL enabled)'), - cfg.StrOpt('kombu_ssl_keyfile', - default='', - help='SSL key file (valid only if SSL enabled)'), - cfg.StrOpt('kombu_ssl_certfile', - default='', - help='SSL cert file (valid only if SSL enabled)'), - cfg.StrOpt('kombu_ssl_ca_certs', - default='', - help=('SSL certification authority file ' - '(valid only if SSL enabled)')), - cfg.StrOpt('rabbit_host', - default='localhost', - help='The RabbitMQ broker address where a single node is used'), - cfg.IntOpt('rabbit_port', - default=5672, - help='The RabbitMQ broker port where a single node is used'), - cfg.ListOpt('rabbit_hosts', - default=['$rabbit_host:$rabbit_port'], - help='RabbitMQ HA cluster host:port pairs'), - cfg.BoolOpt('rabbit_use_ssl', - default=False, - help='connect over SSL for RabbitMQ'), - cfg.StrOpt('rabbit_userid', - default='guest', - help='the RabbitMQ userid'), - cfg.StrOpt('rabbit_password', - default='guest', - help='the RabbitMQ password'), - cfg.StrOpt('rabbit_virtual_host', - default='/', - help='the RabbitMQ virtual host'), - cfg.IntOpt('rabbit_retry_interval', - default=1, - help='how frequently to retry connecting with RabbitMQ'), - cfg.IntOpt('rabbit_retry_backoff', - default=2, - help='how long to backoff for between retries when connecting ' - 'to RabbitMQ'), - cfg.IntOpt('rabbit_max_retries', - default=0, - help='maximum retries with trying to connect to RabbitMQ ' - '(the default of 0 implies an infinite retry count)'), - cfg.BoolOpt('rabbit_durable_queues', - default=False, - help='use durable queues in RabbitMQ'), - cfg.BoolOpt('rabbit_ha_queues', - default=False, - help='use H/A queues in RabbitMQ (x-ha-policy: all).' - 'You need to wipe RabbitMQ database when ' - 'changing this option.'), - -] - -cfg.CONF.register_opts(kombu_opts) - -LOG = rpc_common.LOG - - -def _get_queue_arguments(conf): - """Construct the arguments for declaring a queue. - - If the rabbit_ha_queues option is set, we declare a mirrored queue - as described here: - - http://www.rabbitmq.com/ha.html - - Setting x-ha-policy to all means that the queue will be mirrored - to all nodes in the cluster. - """ - return {'x-ha-policy': 'all'} if conf.rabbit_ha_queues else {} - - -class ConsumerBase(object): - """Consumer base class.""" - - def __init__(self, channel, callback, tag, **kwargs): - """Declare a queue on an amqp channel. - - 'channel' is the amqp channel to use - 'callback' is the callback to call when messages are received - 'tag' is a unique ID for the consumer on the channel - - queue name, exchange name, and other kombu options are - passed in here as a dictionary. - """ - self.callback = callback - self.tag = str(tag) - self.kwargs = kwargs - self.queue = None - self.reconnect(channel) - - def reconnect(self, channel): - """Re-declare the queue after a rabbit reconnect""" - self.channel = channel - self.kwargs['channel'] = channel - self.queue = kombu.entity.Queue(**self.kwargs) - self.queue.declare() - - def consume(self, *args, **kwargs): - """Actually declare the consumer on the amqp channel. This will - start the flow of messages from the queue. Using the - Connection.iterconsume() iterator will process the messages, - calling the appropriate callback. - - If a callback is specified in kwargs, use that. Otherwise, - use the callback passed during __init__() - - If kwargs['nowait'] is True, then this call will block until - a message is read. - - Messages will automatically be acked if the callback doesn't - raise an exception - """ - - options = {'consumer_tag': self.tag} - options['nowait'] = kwargs.get('nowait', False) - callback = kwargs.get('callback', self.callback) - if not callback: - raise ValueError("No callback defined") - - def _callback(raw_message): - message = self.channel.message_to_python(raw_message) - try: - callback(message.payload) - message.ack() - except Exception: - LOG.exception(_("Failed to process message... skipping it.")) - - self.queue.consume(*args, callback=_callback, **options) - - def cancel(self): - """Cancel the consuming from the queue, if it has started""" - try: - self.queue.cancel(self.tag) - except KeyError, e: - # NOTE(comstud): Kludge to get around a amqplib bug - if str(e) != "u'%s'" % self.tag: - raise - self.queue = None - - -class DirectConsumer(ConsumerBase): - """Queue/consumer class for 'direct'""" - - def __init__(self, conf, channel, msg_id, callback, tag, **kwargs): - """Init a 'direct' queue. - - 'channel' is the amqp channel to use - 'msg_id' is the msg_id to listen on - 'callback' is the callback to call when messages are received - 'tag' is a unique ID for the consumer on the channel - - Other kombu options may be passed - """ - # Default options - options = {'durable': False, - 'auto_delete': True, - 'exclusive': True} - options.update(kwargs) - exchange = kombu.entity.Exchange(name=msg_id, - type='direct', - durable=options['durable'], - auto_delete=options['auto_delete']) - super(DirectConsumer, self).__init__(channel, - callback, - tag, - name=msg_id, - exchange=exchange, - routing_key=msg_id, - **options) - - -class TopicConsumer(ConsumerBase): - """Consumer class for 'topic'""" - - def __init__(self, conf, channel, topic, callback, tag, name=None, - exchange_name=None, **kwargs): - """Init a 'topic' queue. - - :param channel: the amqp channel to use - :param topic: the topic to listen on - :paramtype topic: str - :param callback: the callback to call when messages are received - :param tag: a unique ID for the consumer on the channel - :param name: optional queue name, defaults to topic - :paramtype name: str - - Other kombu options may be passed as keyword arguments - """ - # Default options - options = {'durable': conf.rabbit_durable_queues, - 'queue_arguments': _get_queue_arguments(conf), - 'auto_delete': False, - 'exclusive': False} - options.update(kwargs) - exchange_name = exchange_name or rpc_amqp.get_control_exchange(conf) - exchange = kombu.entity.Exchange(name=exchange_name, - type='topic', - durable=options['durable'], - auto_delete=options['auto_delete']) - super(TopicConsumer, self).__init__(channel, - callback, - tag, - name=name or topic, - exchange=exchange, - routing_key=topic, - **options) - - -class FanoutConsumer(ConsumerBase): - """Consumer class for 'fanout'""" - - def __init__(self, conf, channel, topic, callback, tag, **kwargs): - """Init a 'fanout' queue. - - 'channel' is the amqp channel to use - 'topic' is the topic to listen on - 'callback' is the callback to call when messages are received - 'tag' is a unique ID for the consumer on the channel - - Other kombu options may be passed - """ - unique = uuid.uuid4().hex - exchange_name = '%s_fanout' % topic - queue_name = '%s_fanout_%s' % (topic, unique) - - # Default options - options = {'durable': False, - 'queue_arguments': _get_queue_arguments(conf), - 'auto_delete': True, - 'exclusive': True} - options.update(kwargs) - exchange = kombu.entity.Exchange(name=exchange_name, type='fanout', - durable=options['durable'], - auto_delete=options['auto_delete']) - super(FanoutConsumer, self).__init__(channel, callback, tag, - name=queue_name, - exchange=exchange, - routing_key=topic, - **options) - - -class Publisher(object): - """Base Publisher class""" - - def __init__(self, channel, exchange_name, routing_key, **kwargs): - """Init the Publisher class with the exchange_name, routing_key, - and other options - """ - self.exchange_name = exchange_name - self.routing_key = routing_key - self.kwargs = kwargs - self.reconnect(channel) - - def reconnect(self, channel): - """Re-establish the Producer after a rabbit reconnection""" - self.exchange = kombu.entity.Exchange(name=self.exchange_name, - **self.kwargs) - self.producer = kombu.messaging.Producer(exchange=self.exchange, - channel=channel, - routing_key=self.routing_key) - - def send(self, msg): - """Send a message""" - self.producer.publish(msg) - - -class DirectPublisher(Publisher): - """Publisher class for 'direct'""" - def __init__(self, conf, channel, msg_id, **kwargs): - """init a 'direct' publisher. - - Kombu options may be passed as keyword args to override defaults - """ - - options = {'durable': False, - 'auto_delete': True, - 'exclusive': True} - options.update(kwargs) - super(DirectPublisher, self).__init__(channel, msg_id, msg_id, - type='direct', **options) - - -class TopicPublisher(Publisher): - """Publisher class for 'topic'""" - def __init__(self, conf, channel, topic, **kwargs): - """init a 'topic' publisher. - - Kombu options may be passed as keyword args to override defaults - """ - options = {'durable': conf.rabbit_durable_queues, - 'auto_delete': False, - 'exclusive': False} - options.update(kwargs) - exchange_name = rpc_amqp.get_control_exchange(conf) - super(TopicPublisher, self).__init__(channel, - exchange_name, - topic, - type='topic', - **options) - - -class FanoutPublisher(Publisher): - """Publisher class for 'fanout'""" - def __init__(self, conf, channel, topic, **kwargs): - """init a 'fanout' publisher. - - Kombu options may be passed as keyword args to override defaults - """ - options = {'durable': False, - 'auto_delete': True, - 'exclusive': True} - options.update(kwargs) - super(FanoutPublisher, self).__init__(channel, '%s_fanout' % topic, - None, type='fanout', **options) - - -class NotifyPublisher(TopicPublisher): - """Publisher class for 'notify'""" - - def __init__(self, conf, channel, topic, **kwargs): - self.durable = kwargs.pop('durable', conf.rabbit_durable_queues) - self.queue_arguments = _get_queue_arguments(conf) - super(NotifyPublisher, self).__init__(conf, channel, topic, **kwargs) - - def reconnect(self, channel): - super(NotifyPublisher, self).reconnect(channel) - - # NOTE(jerdfelt): Normally the consumer would create the queue, but - # we do this to ensure that messages don't get dropped if the - # consumer is started after we do - queue = kombu.entity.Queue(channel=channel, - exchange=self.exchange, - durable=self.durable, - name=self.routing_key, - routing_key=self.routing_key, - queue_arguments=self.queue_arguments) - queue.declare() - - -class Connection(object): - """Connection object.""" - - pool = None - - def __init__(self, conf, server_params=None): - self.consumers = [] - self.consumer_thread = None - self.conf = conf - self.max_retries = self.conf.rabbit_max_retries - # Try forever? - if self.max_retries <= 0: - self.max_retries = None - self.interval_start = self.conf.rabbit_retry_interval - self.interval_stepping = self.conf.rabbit_retry_backoff - # max retry-interval = 30 seconds - self.interval_max = 30 - self.memory_transport = False - - if server_params is None: - server_params = {} - # Keys to translate from server_params to kombu params - server_params_to_kombu_params = {'username': 'userid'} - - ssl_params = self._fetch_ssl_params() - params_list = [] - for adr in self.conf.rabbit_hosts: - hostname, port = network_utils.parse_host_port( - adr, default_port=self.conf.rabbit_port) - - params = {} - - for sp_key, value in server_params.iteritems(): - p_key = server_params_to_kombu_params.get(sp_key, sp_key) - params[p_key] = value - - params.setdefault('hostname', hostname) - params.setdefault('port', port) - params.setdefault('userid', self.conf.rabbit_userid) - params.setdefault('password', self.conf.rabbit_password) - params.setdefault('virtual_host', self.conf.rabbit_virtual_host) - - if self.conf.fake_rabbit: - params['transport'] = 'memory' - if self.conf.rabbit_use_ssl: - params['ssl'] = ssl_params - - params_list.append(params) - - self.params_list = params_list - - self.memory_transport = self.conf.fake_rabbit - - self.connection = None - self.reconnect() - - def _fetch_ssl_params(self): - """Handles fetching what ssl params - should be used for the connection (if any)""" - ssl_params = dict() - - # http://docs.python.org/library/ssl.html - ssl.wrap_socket - if self.conf.kombu_ssl_version: - ssl_params['ssl_version'] = self.conf.kombu_ssl_version - if self.conf.kombu_ssl_keyfile: - ssl_params['keyfile'] = self.conf.kombu_ssl_keyfile - if self.conf.kombu_ssl_certfile: - ssl_params['certfile'] = self.conf.kombu_ssl_certfile - if self.conf.kombu_ssl_ca_certs: - ssl_params['ca_certs'] = self.conf.kombu_ssl_ca_certs - # We might want to allow variations in the - # future with this? - ssl_params['cert_reqs'] = ssl.CERT_REQUIRED - - if not ssl_params: - # Just have the default behavior - return True - else: - # Return the extended behavior - return ssl_params - - def _connect(self, params): - """Connect to rabbit. Re-establish any queues that may have - been declared before if we are reconnecting. Exceptions should - be handled by the caller. - """ - if self.connection: - LOG.info(_("Reconnecting to AMQP server on " - "%(hostname)s:%(port)d") % params) - try: - self.connection.close() - except self.connection_errors: - pass - # Setting this in case the next statement fails, though - # it shouldn't be doing any network operations, yet. - self.connection = None - self.connection = kombu.connection.BrokerConnection(**params) - self.connection_errors = self.connection.connection_errors - if self.memory_transport: - # Kludge to speed up tests. - self.connection.transport.polling_interval = 0.0 - self.consumer_num = itertools.count(1) - self.connection.connect() - self.channel = self.connection.channel() - # work around 'memory' transport bug in 1.1.3 - if self.memory_transport: - self.channel._new_queue('ae.undeliver') - for consumer in self.consumers: - consumer.reconnect(self.channel) - LOG.info(_('Connected to AMQP server on %(hostname)s:%(port)d') % - params) - - def reconnect(self): - """Handles reconnecting and re-establishing queues. - Will retry up to self.max_retries number of times. - self.max_retries = 0 means to retry forever. - Sleep between tries, starting at self.interval_start - seconds, backing off self.interval_stepping number of seconds - each attempt. - """ - - attempt = 0 - while True: - params = self.params_list[attempt % len(self.params_list)] - attempt += 1 - try: - self._connect(params) - return - except (IOError, self.connection_errors) as e: - pass - except Exception, e: - # NOTE(comstud): Unfortunately it's possible for amqplib - # to return an error not covered by its transport - # connection_errors in the case of a timeout waiting for - # a protocol response. (See paste link in LP888621) - # So, we check all exceptions for 'timeout' in them - # and try to reconnect in this case. - if 'timeout' not in str(e): - raise - - log_info = {} - log_info['err_str'] = str(e) - log_info['max_retries'] = self.max_retries - log_info.update(params) - - if self.max_retries and attempt == self.max_retries: - LOG.error(_('Unable to connect to AMQP server on ' - '%(hostname)s:%(port)d after %(max_retries)d ' - 'tries: %(err_str)s') % log_info) - # NOTE(comstud): Copied from original code. There's - # really no better recourse because if this was a queue we - # need to consume on, we have no way to consume anymore. - sys.exit(1) - - if attempt == 1: - sleep_time = self.interval_start or 1 - elif attempt > 1: - sleep_time += self.interval_stepping - if self.interval_max: - sleep_time = min(sleep_time, self.interval_max) - - log_info['sleep_time'] = sleep_time - LOG.error(_('AMQP server on %(hostname)s:%(port)d is ' - 'unreachable: %(err_str)s. Trying again in ' - '%(sleep_time)d seconds.') % log_info) - time.sleep(sleep_time) - - def ensure(self, error_callback, method, *args, **kwargs): - while True: - try: - return method(*args, **kwargs) - except (self.connection_errors, socket.timeout, IOError), e: - if error_callback: - error_callback(e) - except Exception, e: - # NOTE(comstud): Unfortunately it's possible for amqplib - # to return an error not covered by its transport - # connection_errors in the case of a timeout waiting for - # a protocol response. (See paste link in LP888621) - # So, we check all exceptions for 'timeout' in them - # and try to reconnect in this case. - if 'timeout' not in str(e): - raise - if error_callback: - error_callback(e) - self.reconnect() - - def get_channel(self): - """Convenience call for bin/clear_rabbit_queues""" - return self.channel - - def close(self): - """Close/release this connection""" - self.cancel_consumer_thread() - self.connection.release() - self.connection = None - - def reset(self): - """Reset a connection so it can be used again""" - self.cancel_consumer_thread() - self.channel.close() - self.channel = self.connection.channel() - # work around 'memory' transport bug in 1.1.3 - if self.memory_transport: - self.channel._new_queue('ae.undeliver') - self.consumers = [] - - def declare_consumer(self, consumer_cls, topic, callback): - """Create a Consumer using the class that was passed in and - add it to our list of consumers - """ - - def _connect_error(exc): - log_info = {'topic': topic, 'err_str': str(exc)} - LOG.error(_("Failed to declare consumer for topic '%(topic)s': " - "%(err_str)s") % log_info) - - def _declare_consumer(): - consumer = consumer_cls(self.conf, self.channel, topic, callback, - self.consumer_num.next()) - self.consumers.append(consumer) - return consumer - - return self.ensure(_connect_error, _declare_consumer) - - def iterconsume(self, limit=None, timeout=None): - """Return an iterator that will consume from all queues/consumers""" - - info = {'do_consume': True} - - def _error_callback(exc): - if isinstance(exc, socket.timeout): - LOG.exception(_('Timed out waiting for RPC response: %s') % - str(exc)) - raise rpc_common.Timeout() - else: - LOG.exception(_('Failed to consume message from queue: %s') % - str(exc)) - info['do_consume'] = True - - def _consume(): - if info['do_consume']: - queues_head = self.consumers[:-1] - queues_tail = self.consumers[-1] - for queue in queues_head: - queue.consume(nowait=True) - queues_tail.consume(nowait=False) - info['do_consume'] = False - return self.connection.drain_events(timeout=timeout) - - for iteration in itertools.count(0): - if limit and iteration >= limit: - raise StopIteration - yield self.ensure(_error_callback, _consume) - - def cancel_consumer_thread(self): - """Cancel a consumer thread""" - if self.consumer_thread is not None: - self.consumer_thread.kill() - try: - self.consumer_thread.wait() - except greenlet.GreenletExit: - pass - self.consumer_thread = None - - def publisher_send(self, cls, topic, msg, **kwargs): - """Send to a publisher based on the publisher class""" - - def _error_callback(exc): - log_info = {'topic': topic, 'err_str': str(exc)} - LOG.exception(_("Failed to publish message to topic " - "'%(topic)s': %(err_str)s") % log_info) - - def _publish(): - publisher = cls(self.conf, self.channel, topic, **kwargs) - publisher.send(msg) - - self.ensure(_error_callback, _publish) - - def declare_direct_consumer(self, topic, callback): - """Create a 'direct' queue. - In nova's use, this is generally a msg_id queue used for - responses for call/multicall - """ - self.declare_consumer(DirectConsumer, topic, callback) - - def declare_topic_consumer(self, topic, callback=None, queue_name=None, - exchange_name=None): - """Create a 'topic' consumer.""" - self.declare_consumer(functools.partial(TopicConsumer, - name=queue_name, - exchange_name=exchange_name, - ), - topic, callback) - - def declare_fanout_consumer(self, topic, callback): - """Create a 'fanout' consumer""" - self.declare_consumer(FanoutConsumer, topic, callback) - - def direct_send(self, msg_id, msg): - """Send a 'direct' message""" - self.publisher_send(DirectPublisher, msg_id, msg) - - def topic_send(self, topic, msg): - """Send a 'topic' message""" - self.publisher_send(TopicPublisher, topic, msg) - - def fanout_send(self, topic, msg): - """Send a 'fanout' message""" - self.publisher_send(FanoutPublisher, topic, msg) - - def notify_send(self, topic, msg, **kwargs): - """Send a notify message on a topic""" - self.publisher_send(NotifyPublisher, topic, msg, **kwargs) - - def consume(self, limit=None): - """Consume from all queues/consumers""" - it = self.iterconsume(limit=limit) - while True: - try: - it.next() - except StopIteration: - return - - def _consumer_thread_callback(self): - """ Consumer thread callback used by consume_in_* """ - try: - self.consume() - except greenlet.GreenletExit: - return - - def consume_in_thread(self): - """Consumer from all queues/consumers in a greenthread""" - - if self.consumer_thread is None: - self.consumer_thread = eventlet.spawn( - self._consumer_thread_callback) - return self.consumer_thread - - def consume_in_thread_group(self, thread_group): - """ Consume from all queues/consumers in the supplied ThreadGroup""" - thread_group.add_thread(self._consumer_thread_callback) - - def create_consumer(self, topic, proxy, fanout=False): - """Create a consumer that calls a method in a proxy object""" - proxy_cb = rpc_amqp.ProxyCallback( - self.conf, proxy, - rpc_amqp.get_connection_pool(self.conf, Connection)) - - if fanout: - self.declare_fanout_consumer(topic, proxy_cb) - else: - self.declare_topic_consumer(topic, proxy_cb) - - def create_worker(self, topic, proxy, pool_name): - """Create a worker that calls a method in a proxy object""" - proxy_cb = rpc_amqp.ProxyCallback( - self.conf, proxy, - rpc_amqp.get_connection_pool(self.conf, Connection)) - self.declare_topic_consumer(topic, proxy_cb, pool_name) - - -def create_connection(conf, new=True): - """Create a connection""" - return rpc_amqp.create_connection( - conf, new, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def multicall(conf, context, topic, msg, timeout=None): - """Make a call that returns multiple times.""" - return rpc_amqp.multicall( - conf, context, topic, msg, timeout, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def call(conf, context, topic, msg, timeout=None): - """Sends a message on a topic and wait for a response.""" - return rpc_amqp.call( - conf, context, topic, msg, timeout, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def cast(conf, context, topic, msg): - """Sends a message on a topic without waiting for a response.""" - return rpc_amqp.cast( - conf, context, topic, msg, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def fanout_cast(conf, context, topic, msg): - """Sends a message on a fanout exchange without waiting for a response.""" - return rpc_amqp.fanout_cast( - conf, context, topic, msg, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def cast_to_server(conf, context, server_params, topic, msg): - """Sends a message on a topic to a specific server.""" - return rpc_amqp.cast_to_server( - conf, context, server_params, topic, msg, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def fanout_cast_to_server(conf, context, server_params, topic, msg): - """Sends a message on a fanout exchange to a specific server.""" - return rpc_amqp.fanout_cast_to_server( - conf, context, server_params, topic, msg, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def notify(conf, context, topic, msg): - """Sends a notification event on a topic.""" - return rpc_amqp.notify( - conf, context, topic, msg, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def cleanup(): - return rpc_amqp.cleanup(Connection.pool) diff --git a/bufunfa/openstack/common/rpc/impl_qpid.py b/bufunfa/openstack/common/rpc/impl_qpid.py deleted file mode 100644 index b9b3560..0000000 --- a/bufunfa/openstack/common/rpc/impl_qpid.py +++ /dev/null @@ -1,610 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC -# Copyright 2011 - 2012, Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import functools -import itertools -import logging -import time -import uuid - -import eventlet -import greenlet -import qpid.messaging -import qpid.messaging.exceptions - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import jsonutils -from bufunfa.openstack.common.rpc import amqp as rpc_amqp -from bufunfa.openstack.common.rpc import common as rpc_common - -LOG = logging.getLogger(__name__) - -qpid_opts = [ - cfg.StrOpt('qpid_hostname', - default='localhost', - help='Qpid broker hostname'), - cfg.StrOpt('qpid_port', - default='5672', - help='Qpid broker port'), - cfg.StrOpt('qpid_username', - default='', - help='Username for qpid connection'), - cfg.StrOpt('qpid_password', - default='', - help='Password for qpid connection'), - cfg.StrOpt('qpid_sasl_mechanisms', - default='', - help='Space separated list of SASL mechanisms to use for auth'), - cfg.BoolOpt('qpid_reconnect', - default=True, - help='Automatically reconnect'), - cfg.IntOpt('qpid_reconnect_timeout', - default=0, - help='Reconnection timeout in seconds'), - cfg.IntOpt('qpid_reconnect_limit', - default=0, - help='Max reconnections before giving up'), - cfg.IntOpt('qpid_reconnect_interval_min', - default=0, - help='Minimum seconds between reconnection attempts'), - cfg.IntOpt('qpid_reconnect_interval_max', - default=0, - help='Maximum seconds between reconnection attempts'), - cfg.IntOpt('qpid_reconnect_interval', - default=0, - help='Equivalent to setting max and min to the same value'), - cfg.IntOpt('qpid_heartbeat', - default=60, - help='Seconds between connection keepalive heartbeats'), - cfg.StrOpt('qpid_protocol', - default='tcp', - help="Transport to use, either 'tcp' or 'ssl'"), - cfg.BoolOpt('qpid_tcp_nodelay', - default=True, - help='Disable Nagle algorithm'), -] - -cfg.CONF.register_opts(qpid_opts) - - -class ConsumerBase(object): - """Consumer base class.""" - - def __init__(self, session, callback, node_name, node_opts, - link_name, link_opts): - """Declare a queue on an amqp session. - - 'session' is the amqp session to use - 'callback' is the callback to call when messages are received - 'node_name' is the first part of the Qpid address string, before ';' - 'node_opts' will be applied to the "x-declare" section of "node" - in the address string. - 'link_name' goes into the "name" field of the "link" in the address - string - 'link_opts' will be applied to the "x-declare" section of "link" - in the address string. - """ - self.callback = callback - self.receiver = None - self.session = None - - addr_opts = { - "create": "always", - "node": { - "type": "topic", - "x-declare": { - "durable": True, - "auto-delete": True, - }, - }, - "link": { - "name": link_name, - "durable": True, - "x-declare": { - "durable": False, - "auto-delete": True, - "exclusive": False, - }, - }, - } - addr_opts["node"]["x-declare"].update(node_opts) - addr_opts["link"]["x-declare"].update(link_opts) - - self.address = "%s ; %s" % (node_name, jsonutils.dumps(addr_opts)) - - self.reconnect(session) - - def reconnect(self, session): - """Re-declare the receiver after a qpid reconnect""" - self.session = session - self.receiver = session.receiver(self.address) - self.receiver.capacity = 1 - - def consume(self): - """Fetch the message and pass it to the callback object""" - message = self.receiver.fetch() - try: - self.callback(message.content) - except Exception: - LOG.exception(_("Failed to process message... skipping it.")) - finally: - self.session.acknowledge(message) - - def get_receiver(self): - return self.receiver - - -class DirectConsumer(ConsumerBase): - """Queue/consumer class for 'direct'""" - - def __init__(self, conf, session, msg_id, callback): - """Init a 'direct' queue. - - 'session' is the amqp session to use - 'msg_id' is the msg_id to listen on - 'callback' is the callback to call when messages are received - """ - - super(DirectConsumer, self).__init__(session, callback, - "%s/%s" % (msg_id, msg_id), - {"type": "direct"}, - msg_id, - {"exclusive": True}) - - -class TopicConsumer(ConsumerBase): - """Consumer class for 'topic'""" - - def __init__(self, conf, session, topic, callback, name=None, - exchange_name=None): - """Init a 'topic' queue. - - :param session: the amqp session to use - :param topic: is the topic to listen on - :paramtype topic: str - :param callback: the callback to call when messages are received - :param name: optional queue name, defaults to topic - """ - - exchange_name = exchange_name or rpc_amqp.get_control_exchange(conf) - super(TopicConsumer, self).__init__(session, callback, - "%s/%s" % (exchange_name, topic), - {}, name or topic, {}) - - -class FanoutConsumer(ConsumerBase): - """Consumer class for 'fanout'""" - - def __init__(self, conf, session, topic, callback): - """Init a 'fanout' queue. - - 'session' is the amqp session to use - 'topic' is the topic to listen on - 'callback' is the callback to call when messages are received - """ - - super(FanoutConsumer, self).__init__( - session, callback, - "%s_fanout" % topic, - {"durable": False, "type": "fanout"}, - "%s_fanout_%s" % (topic, uuid.uuid4().hex), - {"exclusive": True}) - - -class Publisher(object): - """Base Publisher class""" - - def __init__(self, session, node_name, node_opts=None): - """Init the Publisher class with the exchange_name, routing_key, - and other options - """ - self.sender = None - self.session = session - - addr_opts = { - "create": "always", - "node": { - "type": "topic", - "x-declare": { - "durable": False, - # auto-delete isn't implemented for exchanges in qpid, - # but put in here anyway - "auto-delete": True, - }, - }, - } - if node_opts: - addr_opts["node"]["x-declare"].update(node_opts) - - self.address = "%s ; %s" % (node_name, jsonutils.dumps(addr_opts)) - - self.reconnect(session) - - def reconnect(self, session): - """Re-establish the Sender after a reconnection""" - self.sender = session.sender(self.address) - - def send(self, msg): - """Send a message""" - self.sender.send(msg) - - -class DirectPublisher(Publisher): - """Publisher class for 'direct'""" - def __init__(self, conf, session, msg_id): - """Init a 'direct' publisher.""" - super(DirectPublisher, self).__init__(session, msg_id, - {"type": "Direct"}) - - -class TopicPublisher(Publisher): - """Publisher class for 'topic'""" - def __init__(self, conf, session, topic): - """init a 'topic' publisher. - """ - exchange_name = rpc_amqp.get_control_exchange(conf) - super(TopicPublisher, self).__init__(session, - "%s/%s" % (exchange_name, topic)) - - -class FanoutPublisher(Publisher): - """Publisher class for 'fanout'""" - def __init__(self, conf, session, topic): - """init a 'fanout' publisher. - """ - super(FanoutPublisher, self).__init__( - session, - "%s_fanout" % topic, {"type": "fanout"}) - - -class NotifyPublisher(Publisher): - """Publisher class for notifications""" - def __init__(self, conf, session, topic): - """init a 'topic' publisher. - """ - exchange_name = rpc_amqp.get_control_exchange(conf) - super(NotifyPublisher, self).__init__(session, - "%s/%s" % (exchange_name, topic), - {"durable": True}) - - -class Connection(object): - """Connection object.""" - - pool = None - - def __init__(self, conf, server_params=None): - self.session = None - self.consumers = {} - self.consumer_thread = None - self.conf = conf - - if server_params is None: - server_params = {} - - default_params = dict(hostname=self.conf.qpid_hostname, - port=self.conf.qpid_port, - username=self.conf.qpid_username, - password=self.conf.qpid_password) - - params = server_params - for key in default_params.keys(): - params.setdefault(key, default_params[key]) - - self.broker = params['hostname'] + ":" + str(params['port']) - # Create the connection - this does not open the connection - self.connection = qpid.messaging.Connection(self.broker) - - # Check if flags are set and if so set them for the connection - # before we call open - self.connection.username = params['username'] - self.connection.password = params['password'] - self.connection.sasl_mechanisms = self.conf.qpid_sasl_mechanisms - self.connection.reconnect = self.conf.qpid_reconnect - if self.conf.qpid_reconnect_timeout: - self.connection.reconnect_timeout = ( - self.conf.qpid_reconnect_timeout) - if self.conf.qpid_reconnect_limit: - self.connection.reconnect_limit = self.conf.qpid_reconnect_limit - if self.conf.qpid_reconnect_interval_max: - self.connection.reconnect_interval_max = ( - self.conf.qpid_reconnect_interval_max) - if self.conf.qpid_reconnect_interval_min: - self.connection.reconnect_interval_min = ( - self.conf.qpid_reconnect_interval_min) - if self.conf.qpid_reconnect_interval: - self.connection.reconnect_interval = ( - self.conf.qpid_reconnect_interval) - self.connection.heartbeat = self.conf.qpid_heartbeat - self.connection.protocol = self.conf.qpid_protocol - self.connection.tcp_nodelay = self.conf.qpid_tcp_nodelay - - # Open is part of reconnect - - # NOTE(WGH) not sure we need this with the reconnect flags - self.reconnect() - - def _register_consumer(self, consumer): - self.consumers[str(consumer.get_receiver())] = consumer - - def _lookup_consumer(self, receiver): - return self.consumers[str(receiver)] - - def reconnect(self): - """Handles reconnecting and re-establishing sessions and queues""" - if self.connection.opened(): - try: - self.connection.close() - except qpid.messaging.exceptions.ConnectionError: - pass - - while True: - try: - self.connection.open() - except qpid.messaging.exceptions.ConnectionError, e: - LOG.error(_('Unable to connect to AMQP server: %s'), e) - time.sleep(self.conf.qpid_reconnect_interval or 1) - else: - break - - LOG.info(_('Connected to AMQP server on %s'), self.broker) - - self.session = self.connection.session() - - for consumer in self.consumers.itervalues(): - consumer.reconnect(self.session) - - if self.consumers: - LOG.debug(_("Re-established AMQP queues")) - - def ensure(self, error_callback, method, *args, **kwargs): - while True: - try: - return method(*args, **kwargs) - except (qpid.messaging.exceptions.Empty, - qpid.messaging.exceptions.ConnectionError), e: - if error_callback: - error_callback(e) - self.reconnect() - - def close(self): - """Close/release this connection""" - self.cancel_consumer_thread() - self.connection.close() - self.connection = None - - def reset(self): - """Reset a connection so it can be used again""" - self.cancel_consumer_thread() - self.session.close() - self.session = self.connection.session() - self.consumers = {} - - def declare_consumer(self, consumer_cls, topic, callback): - """Create a Consumer using the class that was passed in and - add it to our list of consumers - """ - def _connect_error(exc): - log_info = {'topic': topic, 'err_str': str(exc)} - LOG.error(_("Failed to declare consumer for topic '%(topic)s': " - "%(err_str)s") % log_info) - - def _declare_consumer(): - consumer = consumer_cls(self.conf, self.session, topic, callback) - self._register_consumer(consumer) - return consumer - - return self.ensure(_connect_error, _declare_consumer) - - def iterconsume(self, limit=None, timeout=None): - """Return an iterator that will consume from all queues/consumers""" - - def _error_callback(exc): - if isinstance(exc, qpid.messaging.exceptions.Empty): - LOG.exception(_('Timed out waiting for RPC response: %s') % - str(exc)) - raise rpc_common.Timeout() - else: - LOG.exception(_('Failed to consume message from queue: %s') % - str(exc)) - - def _consume(): - nxt_receiver = self.session.next_receiver(timeout=timeout) - try: - self._lookup_consumer(nxt_receiver).consume() - except Exception: - LOG.exception(_("Error processing message. Skipping it.")) - - for iteration in itertools.count(0): - if limit and iteration >= limit: - raise StopIteration - yield self.ensure(_error_callback, _consume) - - def cancel_consumer_thread(self): - """Cancel a consumer thread""" - if self.consumer_thread is not None: - self.consumer_thread.kill() - try: - self.consumer_thread.wait() - except greenlet.GreenletExit: - pass - self.consumer_thread = None - - def publisher_send(self, cls, topic, msg): - """Send to a publisher based on the publisher class""" - - def _connect_error(exc): - log_info = {'topic': topic, 'err_str': str(exc)} - LOG.exception(_("Failed to publish message to topic " - "'%(topic)s': %(err_str)s") % log_info) - - def _publisher_send(): - publisher = cls(self.conf, self.session, topic) - publisher.send(msg) - - return self.ensure(_connect_error, _publisher_send) - - def declare_direct_consumer(self, topic, callback): - """Create a 'direct' queue. - In nova's use, this is generally a msg_id queue used for - responses for call/multicall - """ - self.declare_consumer(DirectConsumer, topic, callback) - - def declare_topic_consumer(self, topic, callback=None, queue_name=None, - exchange_name=None): - """Create a 'topic' consumer.""" - self.declare_consumer(functools.partial(TopicConsumer, - name=queue_name, - exchange_name=exchange_name, - ), - topic, callback) - - def declare_fanout_consumer(self, topic, callback): - """Create a 'fanout' consumer""" - self.declare_consumer(FanoutConsumer, topic, callback) - - def direct_send(self, msg_id, msg): - """Send a 'direct' message""" - self.publisher_send(DirectPublisher, msg_id, msg) - - def topic_send(self, topic, msg): - """Send a 'topic' message""" - self.publisher_send(TopicPublisher, topic, msg) - - def fanout_send(self, topic, msg): - """Send a 'fanout' message""" - self.publisher_send(FanoutPublisher, topic, msg) - - def notify_send(self, topic, msg, **kwargs): - """Send a notify message on a topic""" - self.publisher_send(NotifyPublisher, topic, msg) - - def _consumer_thread_callback(self): - """ Consumer thread callback used by consume_in_* """ - try: - self.consume() - except greenlet.GreenletExit: - return - - def consume(self, limit=None): - """Consume from all queues/consumers""" - it = self.iterconsume(limit=limit) - while True: - try: - it.next() - except StopIteration: - return - - def consume_in_thread(self): - """Consumer from all queues/consumers in a greenthread""" - - if self.consumer_thread is None: - self.consumer_thread = eventlet.spawn( - self._consumer_thread_callback) - return self.consumer_thread - - def consume_in_thread_group(self, thread_group): - """ Consume from all queues/consumers in the supplied ThreadGroup""" - thread_group.add_thread(self._consumer_thread_callback) - - def create_consumer(self, topic, proxy, fanout=False): - """Create a consumer that calls a method in a proxy object""" - proxy_cb = rpc_amqp.ProxyCallback( - self.conf, proxy, - rpc_amqp.get_connection_pool(self.conf, Connection)) - - if fanout: - consumer = FanoutConsumer(self.conf, self.session, topic, proxy_cb) - else: - consumer = TopicConsumer(self.conf, self.session, topic, proxy_cb) - - self._register_consumer(consumer) - - return consumer - - def create_worker(self, topic, proxy, pool_name): - """Create a worker that calls a method in a proxy object""" - proxy_cb = rpc_amqp.ProxyCallback( - self.conf, proxy, - rpc_amqp.get_connection_pool(self.conf, Connection)) - - consumer = TopicConsumer(self.conf, self.session, topic, proxy_cb, - name=pool_name) - - self._register_consumer(consumer) - - return consumer - - -def create_connection(conf, new=True): - """Create a connection""" - return rpc_amqp.create_connection( - conf, new, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def multicall(conf, context, topic, msg, timeout=None): - """Make a call that returns multiple times.""" - return rpc_amqp.multicall( - conf, context, topic, msg, timeout, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def call(conf, context, topic, msg, timeout=None): - """Sends a message on a topic and wait for a response.""" - return rpc_amqp.call( - conf, context, topic, msg, timeout, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def cast(conf, context, topic, msg): - """Sends a message on a topic without waiting for a response.""" - return rpc_amqp.cast( - conf, context, topic, msg, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def fanout_cast(conf, context, topic, msg): - """Sends a message on a fanout exchange without waiting for a response.""" - return rpc_amqp.fanout_cast( - conf, context, topic, msg, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def cast_to_server(conf, context, server_params, topic, msg): - """Sends a message on a topic to a specific server.""" - return rpc_amqp.cast_to_server( - conf, context, server_params, topic, msg, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def fanout_cast_to_server(conf, context, server_params, topic, msg): - """Sends a message on a fanout exchange to a specific server.""" - return rpc_amqp.fanout_cast_to_server( - conf, context, server_params, topic, msg, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def notify(conf, context, topic, msg): - """Sends a notification event on a topic.""" - return rpc_amqp.notify(conf, context, topic, msg, - rpc_amqp.get_connection_pool(conf, Connection)) - - -def cleanup(): - return rpc_amqp.cleanup(Connection.pool) diff --git a/bufunfa/openstack/common/rpc/impl_zmq.py b/bufunfa/openstack/common/rpc/impl_zmq.py deleted file mode 100644 index 702bf82..0000000 --- a/bufunfa/openstack/common/rpc/impl_zmq.py +++ /dev/null @@ -1,728 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Cloudscaling Group, Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pprint -import socket -import string -import sys -import types -import uuid - -import eventlet -from eventlet.green import zmq -import greenlet - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import importutils -from bufunfa.openstack.common import jsonutils -from bufunfa.openstack.common.rpc import common as rpc_common - - -# for convenience, are not modified. -pformat = pprint.pformat -Timeout = eventlet.timeout.Timeout -LOG = rpc_common.LOG -RemoteError = rpc_common.RemoteError -RPCException = rpc_common.RPCException - -zmq_opts = [ - cfg.StrOpt('rpc_zmq_bind_address', default='*', - help='ZeroMQ bind address. Should be a wildcard (*), ' - 'an ethernet interface, or IP. ' - 'The "host" option should point or resolve to this ' - 'address.'), - - # The module.Class to use for matchmaking. - cfg.StrOpt( - 'rpc_zmq_matchmaker', - default=('bufunfa.openstack.common.rpc.' - 'matchmaker.MatchMakerLocalhost'), - help='MatchMaker driver', - ), - - # The following port is unassigned by IANA as of 2012-05-21 - cfg.IntOpt('rpc_zmq_port', default=9501, - help='ZeroMQ receiver listening port'), - - cfg.IntOpt('rpc_zmq_contexts', default=1, - help='Number of ZeroMQ contexts, defaults to 1'), - - cfg.StrOpt('rpc_zmq_ipc_dir', default='/var/run/openstack', - help='Directory for holding IPC sockets'), - - cfg.StrOpt('rpc_zmq_host', default=socket.gethostname(), - help='Name of this node. Must be a valid hostname, FQDN, or ' - 'IP address. Must match "host" option, if running Nova.') -] - - -# These globals are defined in register_opts(conf), -# a mandatory initialization call -CONF = None -ZMQ_CTX = None # ZeroMQ Context, must be global. -matchmaker = None # memoized matchmaker object - - -def _serialize(data): - """ - Serialization wrapper - We prefer using JSON, but it cannot encode all types. - Error if a developer passes us bad data. - """ - try: - return str(jsonutils.dumps(data, ensure_ascii=True)) - except TypeError: - LOG.error(_("JSON serialization failed.")) - raise - - -def _deserialize(data): - """ - Deserialization wrapper - """ - LOG.debug(_("Deserializing: %s"), data) - return jsonutils.loads(data) - - -class ZmqSocket(object): - """ - A tiny wrapper around ZeroMQ to simplify the send/recv protocol - and connection management. - - Can be used as a Context (supports the 'with' statement). - """ - - def __init__(self, addr, zmq_type, bind=True, subscribe=None): - self.sock = ZMQ_CTX.socket(zmq_type) - self.addr = addr - self.type = zmq_type - self.subscriptions = [] - - # Support failures on sending/receiving on wrong socket type. - self.can_recv = zmq_type in (zmq.PULL, zmq.SUB) - self.can_send = zmq_type in (zmq.PUSH, zmq.PUB) - self.can_sub = zmq_type in (zmq.SUB, ) - - # Support list, str, & None for subscribe arg (cast to list) - do_sub = { - list: subscribe, - str: [subscribe], - type(None): [] - }[type(subscribe)] - - for f in do_sub: - self.subscribe(f) - - str_data = {'addr': addr, 'type': self.socket_s(), - 'subscribe': subscribe, 'bind': bind} - - LOG.debug(_("Connecting to %(addr)s with %(type)s"), str_data) - LOG.debug(_("-> Subscribed to %(subscribe)s"), str_data) - LOG.debug(_("-> bind: %(bind)s"), str_data) - - try: - if bind: - self.sock.bind(addr) - else: - self.sock.connect(addr) - except Exception: - raise RPCException(_("Could not open socket.")) - - def socket_s(self): - """Get socket type as string.""" - t_enum = ('PUSH', 'PULL', 'PUB', 'SUB', 'REP', 'REQ', 'ROUTER', - 'DEALER') - return dict(map(lambda t: (getattr(zmq, t), t), t_enum))[self.type] - - def subscribe(self, msg_filter): - """Subscribe.""" - if not self.can_sub: - raise RPCException("Cannot subscribe on this socket.") - LOG.debug(_("Subscribing to %s"), msg_filter) - - try: - self.sock.setsockopt(zmq.SUBSCRIBE, msg_filter) - except Exception: - return - - self.subscriptions.append(msg_filter) - - def unsubscribe(self, msg_filter): - """Unsubscribe.""" - if msg_filter not in self.subscriptions: - return - self.sock.setsockopt(zmq.UNSUBSCRIBE, msg_filter) - self.subscriptions.remove(msg_filter) - - def close(self): - if self.sock is None or self.sock.closed: - return - - # We must unsubscribe, or we'll leak descriptors. - if len(self.subscriptions) > 0: - for f in self.subscriptions: - try: - self.sock.setsockopt(zmq.UNSUBSCRIBE, f) - except Exception: - pass - self.subscriptions = [] - - # Linger -1 prevents lost/dropped messages - try: - self.sock.close(linger=-1) - except Exception: - pass - self.sock = None - - def recv(self): - if not self.can_recv: - raise RPCException(_("You cannot recv on this socket.")) - return self.sock.recv_multipart() - - def send(self, data): - if not self.can_send: - raise RPCException(_("You cannot send on this socket.")) - self.sock.send_multipart(data) - - -class ZmqClient(object): - """Client for ZMQ sockets.""" - - def __init__(self, addr, socket_type=zmq.PUSH, bind=False): - self.outq = ZmqSocket(addr, socket_type, bind=bind) - - def cast(self, msg_id, topic, data): - self.outq.send([str(msg_id), str(topic), str('cast'), - _serialize(data)]) - - def close(self): - self.outq.close() - - -class RpcContext(rpc_common.CommonRpcContext): - """Context that supports replying to a rpc.call.""" - def __init__(self, **kwargs): - self.replies = [] - super(RpcContext, self).__init__(**kwargs) - - def deepcopy(self): - values = self.to_dict() - values['replies'] = self.replies - return self.__class__(**values) - - def reply(self, reply=None, failure=None, ending=False): - if ending: - return - self.replies.append(reply) - - @classmethod - def marshal(self, ctx): - ctx_data = ctx.to_dict() - return _serialize(ctx_data) - - @classmethod - def unmarshal(self, data): - return RpcContext.from_dict(_deserialize(data)) - - -class InternalContext(object): - """Used by ConsumerBase as a private context for - methods.""" - - def __init__(self, proxy): - self.proxy = proxy - self.msg_waiter = None - - def _get_response(self, ctx, proxy, topic, data): - """Process a curried message and cast the result to topic.""" - LOG.debug(_("Running func with context: %s"), ctx.to_dict()) - data.setdefault('version', None) - data.setdefault('args', []) - - try: - result = proxy.dispatch( - ctx, data['version'], data['method'], **data['args']) - return ConsumerBase.normalize_reply(result, ctx.replies) - except greenlet.GreenletExit: - # ignore these since they are just from shutdowns - pass - except Exception: - return {'exc': - rpc_common.serialize_remote_exception(sys.exc_info())} - - def reply(self, ctx, proxy, - msg_id=None, context=None, topic=None, msg=None): - """Reply to a casted call.""" - # Our real method is curried into msg['args'] - - child_ctx = RpcContext.unmarshal(msg[0]) - response = ConsumerBase.normalize_reply( - self._get_response(child_ctx, proxy, topic, msg[1]), - ctx.replies) - - LOG.debug(_("Sending reply")) - cast(CONF, ctx, topic, { - 'method': '-process_reply', - 'args': { - 'msg_id': msg_id, - 'response': response - } - }) - - -class ConsumerBase(object): - """Base Consumer.""" - - def __init__(self): - self.private_ctx = InternalContext(None) - - @classmethod - def normalize_reply(self, result, replies): - #TODO(ewindisch): re-evaluate and document this method. - if isinstance(result, types.GeneratorType): - return list(result) - elif replies: - return replies - else: - return [result] - - def process(self, style, target, proxy, ctx, data): - # Method starting with - are - # processed internally. (non-valid method name) - method = data['method'] - - # Internal method - # uses internal context for safety. - if data['method'][0] == '-': - # For reply / process_reply - method = method[1:] - if method == 'reply': - self.private_ctx.reply(ctx, proxy, **data['args']) - return - - data.setdefault('version', None) - data.setdefault('args', []) - proxy.dispatch(ctx, data['version'], - data['method'], **data['args']) - - -class ZmqBaseReactor(ConsumerBase): - """ - A consumer class implementing a - centralized casting broker (PULL-PUSH) - for RoundRobin requests. - """ - - def __init__(self, conf): - super(ZmqBaseReactor, self).__init__() - - self.mapping = {} - self.proxies = {} - self.threads = [] - self.sockets = [] - self.subscribe = {} - - self.pool = eventlet.greenpool.GreenPool(conf.rpc_thread_pool_size) - - def register(self, proxy, in_addr, zmq_type_in, out_addr=None, - zmq_type_out=None, in_bind=True, out_bind=True, - subscribe=None): - - LOG.info(_("Registering reactor")) - - if zmq_type_in not in (zmq.PULL, zmq.SUB): - raise RPCException("Bad input socktype") - - # Items push in. - inq = ZmqSocket(in_addr, zmq_type_in, bind=in_bind, - subscribe=subscribe) - - self.proxies[inq] = proxy - self.sockets.append(inq) - - LOG.info(_("In reactor registered")) - - if not out_addr: - return - - if zmq_type_out not in (zmq.PUSH, zmq.PUB): - raise RPCException("Bad output socktype") - - # Items push out. - outq = ZmqSocket(out_addr, zmq_type_out, bind=out_bind) - - self.mapping[inq] = outq - self.mapping[outq] = inq - self.sockets.append(outq) - - LOG.info(_("Out reactor registered")) - - def _consumer_thread_callback(self, sock): - """ Consumer thread callback used by consume_in_* """ - - LOG.info(_("Consuming socket")) - while True: - self.consume(sock) - - def consume_in_thread(self): - for k in self.proxies.keys(): - self.threads.append( - self.pool.spawn(self._consumer_thread_callback, k) - ) - - def consume_in_thread_group(self, thread_group): - """ Consume from all queues/consumers in the supplied ThreadGroup""" - for k in self.proxies.keys(): - thread_group.add_thread(self._consumer_thread_callback, k) - - def wait(self): - for t in self.threads: - t.wait() - - def close(self): - for s in self.sockets: - s.close() - - for t in self.threads: - t.kill() - - -class ZmqProxy(ZmqBaseReactor): - """ - A consumer class implementing a - topic-based proxy, forwarding to - IPC sockets. - """ - - def __init__(self, conf): - super(ZmqProxy, self).__init__(conf) - - self.topic_proxy = {} - ipc_dir = CONF.rpc_zmq_ipc_dir - - self.topic_proxy['zmq_replies'] = \ - ZmqSocket("ipc://%s/zmq_topic_zmq_replies" % (ipc_dir, ), - zmq.PUB, bind=True) - self.sockets.append(self.topic_proxy['zmq_replies']) - - def consume(self, sock): - ipc_dir = CONF.rpc_zmq_ipc_dir - - #TODO(ewindisch): use zero-copy (i.e. references, not copying) - data = sock.recv() - msg_id, topic, style, in_msg = data - topic = topic.split('.', 1)[0] - - LOG.debug(_("CONSUMER GOT %s"), ' '.join(map(pformat, data))) - - # Handle zmq_replies magic - if topic.startswith('fanout~'): - sock_type = zmq.PUB - elif topic.startswith('zmq_replies'): - sock_type = zmq.PUB - inside = _deserialize(in_msg) - msg_id = inside[-1]['args']['msg_id'] - response = inside[-1]['args']['response'] - LOG.debug(_("->response->%s"), response) - data = [str(msg_id), _serialize(response)] - else: - sock_type = zmq.PUSH - - if not topic in self.topic_proxy: - outq = ZmqSocket("ipc://%s/zmq_topic_%s" % (ipc_dir, topic), - sock_type, bind=True) - self.topic_proxy[topic] = outq - self.sockets.append(outq) - LOG.info(_("Created topic proxy: %s"), topic) - - # It takes some time for a pub socket to open, - # before we can have any faith in doing a send() to it. - if sock_type == zmq.PUB: - eventlet.sleep(.5) - - LOG.debug(_("ROUTER RELAY-OUT START %(data)s") % {'data': data}) - self.topic_proxy[topic].send(data) - LOG.debug(_("ROUTER RELAY-OUT SUCCEEDED %(data)s") % {'data': data}) - - -class ZmqReactor(ZmqBaseReactor): - """ - A consumer class implementing a - consumer for messages. Can also be - used as a 1:1 proxy - """ - - def __init__(self, conf): - super(ZmqReactor, self).__init__(conf) - - def consume(self, sock): - #TODO(ewindisch): use zero-copy (i.e. references, not copying) - data = sock.recv() - LOG.debug(_("CONSUMER RECEIVED DATA: %s"), data) - if sock in self.mapping: - LOG.debug(_("ROUTER RELAY-OUT %(data)s") % { - 'data': data}) - self.mapping[sock].send(data) - return - - msg_id, topic, style, in_msg = data - - ctx, request = _deserialize(in_msg) - ctx = RpcContext.unmarshal(ctx) - - proxy = self.proxies[sock] - - self.pool.spawn_n(self.process, style, topic, - proxy, ctx, request) - - -class Connection(rpc_common.Connection): - """Manages connections and threads.""" - - def __init__(self, conf): - self.reactor = ZmqReactor(conf) - - def create_consumer(self, topic, proxy, fanout=False): - # Only consume on the base topic name. - topic = topic.split('.', 1)[0] - - LOG.info(_("Create Consumer for topic (%(topic)s)") % - {'topic': topic}) - - # Subscription scenarios - if fanout: - subscribe = ('', fanout)[type(fanout) == str] - sock_type = zmq.SUB - topic = 'fanout~' + topic - else: - sock_type = zmq.PULL - subscribe = None - - # Receive messages from (local) proxy - inaddr = "ipc://%s/zmq_topic_%s" % \ - (CONF.rpc_zmq_ipc_dir, topic) - - LOG.debug(_("Consumer is a zmq.%s"), - ['PULL', 'SUB'][sock_type == zmq.SUB]) - - self.reactor.register(proxy, inaddr, sock_type, - subscribe=subscribe, in_bind=False) - - def close(self): - self.reactor.close() - - def wait(self): - self.reactor.wait() - - def consume_in_thread(self): - self.reactor.consume_in_thread() - - def consume_in_thread_group(self, thread_group): - self.reactor.consume_in_thread_group(thread_group) - - -def _cast(addr, context, msg_id, topic, msg, timeout=None): - timeout_cast = timeout or CONF.rpc_cast_timeout - payload = [RpcContext.marshal(context), msg] - - with Timeout(timeout_cast, exception=rpc_common.Timeout): - try: - conn = ZmqClient(addr) - - # assumes cast can't return an exception - conn.cast(msg_id, topic, payload) - except zmq.ZMQError: - raise RPCException("Cast failed. ZMQ Socket Exception") - finally: - if 'conn' in vars(): - conn.close() - - -def _call(addr, context, msg_id, topic, msg, timeout=None): - # timeout_response is how long we wait for a response - timeout = timeout or CONF.rpc_response_timeout - - # The msg_id is used to track replies. - msg_id = uuid.uuid4().hex - - # Replies always come into the reply service. - reply_topic = "zmq_replies.%s" % CONF.rpc_zmq_host - - LOG.debug(_("Creating payload")) - # Curry the original request into a reply method. - mcontext = RpcContext.marshal(context) - payload = { - 'method': '-reply', - 'args': { - 'msg_id': msg_id, - 'context': mcontext, - 'topic': reply_topic, - 'msg': [mcontext, msg] - } - } - - LOG.debug(_("Creating queue socket for reply waiter")) - - # Messages arriving async. - # TODO(ewindisch): have reply consumer with dynamic subscription mgmt - with Timeout(timeout, exception=rpc_common.Timeout): - try: - msg_waiter = ZmqSocket( - "ipc://%s/zmq_topic_zmq_replies" % CONF.rpc_zmq_ipc_dir, - zmq.SUB, subscribe=msg_id, bind=False - ) - - LOG.debug(_("Sending cast")) - _cast(addr, context, msg_id, topic, payload) - - LOG.debug(_("Cast sent; Waiting reply")) - # Blocks until receives reply - msg = msg_waiter.recv() - LOG.debug(_("Received message: %s"), msg) - LOG.debug(_("Unpacking response")) - responses = _deserialize(msg[-1]) - # ZMQError trumps the Timeout error. - except zmq.ZMQError: - raise RPCException("ZMQ Socket Error") - finally: - if 'msg_waiter' in vars(): - msg_waiter.close() - - # It seems we don't need to do all of the following, - # but perhaps it would be useful for multicall? - # One effect of this is that we're checking all - # responses for Exceptions. - for resp in responses: - if isinstance(resp, types.DictType) and 'exc' in resp: - raise rpc_common.deserialize_remote_exception(CONF, resp['exc']) - - return responses[-1] - - -def _multi_send(method, context, topic, msg, timeout=None): - """ - Wraps the sending of messages, - dispatches to the matchmaker and sends - message to all relevant hosts. - """ - conf = CONF - LOG.debug(_("%(msg)s") % {'msg': ' '.join(map(pformat, (topic, msg)))}) - - queues = matchmaker.queues(topic) - LOG.debug(_("Sending message(s) to: %s"), queues) - - # Don't stack if we have no matchmaker results - if len(queues) == 0: - LOG.warn(_("No matchmaker results. Not casting.")) - # While not strictly a timeout, callers know how to handle - # this exception and a timeout isn't too big a lie. - raise rpc_common.Timeout, "No match from matchmaker." - - # This supports brokerless fanout (addresses > 1) - for queue in queues: - (_topic, ip_addr) = queue - _addr = "tcp://%s:%s" % (ip_addr, conf.rpc_zmq_port) - - if method.__name__ == '_cast': - eventlet.spawn_n(method, _addr, context, - _topic, _topic, msg, timeout) - return - return method(_addr, context, _topic, _topic, msg, timeout) - - -def create_connection(conf, new=True): - return Connection(conf) - - -def multicall(conf, *args, **kwargs): - """Multiple calls.""" - return _multi_send(_call, *args, **kwargs) - - -def call(conf, *args, **kwargs): - """Send a message, expect a response.""" - data = _multi_send(_call, *args, **kwargs) - return data[-1] - - -def cast(conf, *args, **kwargs): - """Send a message expecting no reply.""" - _multi_send(_cast, *args, **kwargs) - - -def fanout_cast(conf, context, topic, msg, **kwargs): - """Send a message to all listening and expect no reply.""" - # NOTE(ewindisch): fanout~ is used because it avoid splitting on . - # and acts as a non-subtle hint to the matchmaker and ZmqProxy. - _multi_send(_cast, context, 'fanout~' + str(topic), msg, **kwargs) - - -def notify(conf, context, topic, msg, **kwargs): - """ - Send notification event. - Notifications are sent to topic-priority. - This differs from the AMQP drivers which send to topic.priority. - """ - # NOTE(ewindisch): dot-priority in rpc notifier does not - # work with our assumptions. - topic.replace('.', '-') - cast(conf, context, topic, msg, **kwargs) - - -def cleanup(): - """Clean up resources in use by implementation.""" - global ZMQ_CTX - global matchmaker - matchmaker = None - ZMQ_CTX.term() - ZMQ_CTX = None - - -def register_opts(conf): - """Registration of options for this driver.""" - #NOTE(ewindisch): ZMQ_CTX and matchmaker - # are initialized here as this is as good - # an initialization method as any. - - # We memoize through these globals - global ZMQ_CTX - global matchmaker - global CONF - - if not CONF: - conf.register_opts(zmq_opts) - CONF = conf - # Don't re-set, if this method is called twice. - if not ZMQ_CTX: - ZMQ_CTX = zmq.Context(conf.rpc_zmq_contexts) - if not matchmaker: - # rpc_zmq_matchmaker should be set to a 'module.Class' - mm_path = conf.rpc_zmq_matchmaker.split('.') - mm_module = '.'.join(mm_path[:-1]) - mm_class = mm_path[-1] - - # Only initialize a class. - if mm_path[-1][0] not in string.ascii_uppercase: - LOG.error(_("Matchmaker could not be loaded.\n" - "rpc_zmq_matchmaker is not a class.")) - raise RPCException(_("Error loading Matchmaker.")) - - mm_impl = importutils.import_module(mm_module) - mm_constructor = getattr(mm_impl, mm_class) - matchmaker = mm_constructor() - - -register_opts(cfg.CONF) diff --git a/bufunfa/openstack/common/rpc/matchmaker.py b/bufunfa/openstack/common/rpc/matchmaker.py deleted file mode 100644 index d77f478..0000000 --- a/bufunfa/openstack/common/rpc/matchmaker.py +++ /dev/null @@ -1,258 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Cloudscaling Group, Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -The MatchMaker classes should except a Topic or Fanout exchange key and -return keys for direct exchanges, per (approximate) AMQP parlance. -""" - -import contextlib -import itertools -import json -import logging - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common.gettextutils import _ - - -matchmaker_opts = [ - # Matchmaker ring file - cfg.StrOpt('matchmaker_ringfile', - default='/etc/nova/matchmaker_ring.json', - help='Matchmaker ring file (JSON)'), -] - -CONF = cfg.CONF -CONF.register_opts(matchmaker_opts) -LOG = logging.getLogger(__name__) -contextmanager = contextlib.contextmanager - - -class MatchMakerException(Exception): - """Signified a match could not be found.""" - message = _("Match not found by MatchMaker.") - - -class Exchange(object): - """ - Implements lookups. - Subclass this to support hashtables, dns, etc. - """ - def __init__(self): - pass - - def run(self, key): - raise NotImplementedError() - - -class Binding(object): - """ - A binding on which to perform a lookup. - """ - def __init__(self): - pass - - def test(self, key): - raise NotImplementedError() - - -class MatchMakerBase(object): - """Match Maker Base Class.""" - - def __init__(self): - # Array of tuples. Index [2] toggles negation, [3] is last-if-true - self.bindings = [] - - def add_binding(self, binding, rule, last=True): - self.bindings.append((binding, rule, False, last)) - - #NOTE(ewindisch): kept the following method in case we implement the - # underlying support. - #def add_negate_binding(self, binding, rule, last=True): - # self.bindings.append((binding, rule, True, last)) - - def queues(self, key): - workers = [] - - # bit is for negate bindings - if we choose to implement it. - # last stops processing rules if this matches. - for (binding, exchange, bit, last) in self.bindings: - if binding.test(key): - workers.extend(exchange.run(key)) - - # Support last. - if last: - return workers - return workers - - -class DirectBinding(Binding): - """ - Specifies a host in the key via a '.' character - Although dots are used in the key, the behavior here is - that it maps directly to a host, thus direct. - """ - def test(self, key): - if '.' in key: - return True - return False - - -class TopicBinding(Binding): - """ - Where a 'bare' key without dots. - AMQP generally considers topic exchanges to be those *with* dots, - but we deviate here in terminology as the behavior here matches - that of a topic exchange (whereas where there are dots, behavior - matches that of a direct exchange. - """ - def test(self, key): - if '.' not in key: - return True - return False - - -class FanoutBinding(Binding): - """Match on fanout keys, where key starts with 'fanout.' string.""" - def test(self, key): - if key.startswith('fanout~'): - return True - return False - - -class StubExchange(Exchange): - """Exchange that does nothing.""" - def run(self, key): - return [(key, None)] - - -class RingExchange(Exchange): - """ - Match Maker where hosts are loaded from a static file containing - a hashmap (JSON formatted). - - __init__ takes optional ring dictionary argument, otherwise - loads the ringfile from CONF.mathcmaker_ringfile. - """ - def __init__(self, ring=None): - super(RingExchange, self).__init__() - - if ring: - self.ring = ring - else: - fh = open(CONF.matchmaker_ringfile, 'r') - self.ring = json.load(fh) - fh.close() - - self.ring0 = {} - for k in self.ring.keys(): - self.ring0[k] = itertools.cycle(self.ring[k]) - - def _ring_has(self, key): - if key in self.ring0: - return True - return False - - -class RoundRobinRingExchange(RingExchange): - """A Topic Exchange based on a hashmap.""" - def __init__(self, ring=None): - super(RoundRobinRingExchange, self).__init__(ring) - - def run(self, key): - if not self._ring_has(key): - LOG.warn( - _("No key defining hosts for topic '%s', " - "see ringfile") % (key, ) - ) - return [] - host = next(self.ring0[key]) - return [(key + '.' + host, host)] - - -class FanoutRingExchange(RingExchange): - """Fanout Exchange based on a hashmap.""" - def __init__(self, ring=None): - super(FanoutRingExchange, self).__init__(ring) - - def run(self, key): - # Assume starts with "fanout~", strip it for lookup. - nkey = key.split('fanout~')[1:][0] - if not self._ring_has(nkey): - LOG.warn( - _("No key defining hosts for topic '%s', " - "see ringfile") % (nkey, ) - ) - return [] - return map(lambda x: (key + '.' + x, x), self.ring[nkey]) - - -class LocalhostExchange(Exchange): - """Exchange where all direct topics are local.""" - def __init__(self): - super(Exchange, self).__init__() - - def run(self, key): - return [(key.split('.')[0] + '.localhost', 'localhost')] - - -class DirectExchange(Exchange): - """ - Exchange where all topic keys are split, sending to second half. - i.e. "compute.host" sends a message to "compute" running on "host" - """ - def __init__(self): - super(Exchange, self).__init__() - - def run(self, key): - b, e = key.split('.', 1) - return [(b, e)] - - -class MatchMakerRing(MatchMakerBase): - """ - Match Maker where hosts are loaded from a static hashmap. - """ - def __init__(self, ring=None): - super(MatchMakerRing, self).__init__() - self.add_binding(FanoutBinding(), FanoutRingExchange(ring)) - self.add_binding(DirectBinding(), DirectExchange()) - self.add_binding(TopicBinding(), RoundRobinRingExchange(ring)) - - -class MatchMakerLocalhost(MatchMakerBase): - """ - Match Maker where all bare topics resolve to localhost. - Useful for testing. - """ - def __init__(self): - super(MatchMakerLocalhost, self).__init__() - self.add_binding(FanoutBinding(), LocalhostExchange()) - self.add_binding(DirectBinding(), DirectExchange()) - self.add_binding(TopicBinding(), LocalhostExchange()) - - -class MatchMakerStub(MatchMakerBase): - """ - Match Maker where topics are untouched. - Useful for testing, or for AMQP/brokered queues. - Will not work where knowledge of hosts is known (i.e. zeromq) - """ - def __init__(self): - super(MatchMakerLocalhost, self).__init__() - - self.add_binding(FanoutBinding(), StubExchange()) - self.add_binding(DirectBinding(), StubExchange()) - self.add_binding(TopicBinding(), StubExchange()) diff --git a/bufunfa/openstack/common/rpc/proxy.py b/bufunfa/openstack/common/rpc/proxy.py deleted file mode 100644 index e659878..0000000 --- a/bufunfa/openstack/common/rpc/proxy.py +++ /dev/null @@ -1,165 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -A helper class for proxy objects to remote APIs. - -For more information about rpc API version numbers, see: - rpc/dispatcher.py -""" - - -from bufunfa.openstack.common import rpc - - -class RpcProxy(object): - """A helper class for rpc clients. - - This class is a wrapper around the RPC client API. It allows you to - specify the topic and API version in a single place. This is intended to - be used as a base class for a class that implements the client side of an - rpc API. - """ - - def __init__(self, topic, default_version): - """Initialize an RpcProxy. - - :param topic: The topic to use for all messages. - :param default_version: The default API version to request in all - outgoing messages. This can be overridden on a per-message - basis. - """ - self.topic = topic - self.default_version = default_version - super(RpcProxy, self).__init__() - - def _set_version(self, msg, vers): - """Helper method to set the version in a message. - - :param msg: The message having a version added to it. - :param vers: The version number to add to the message. - """ - msg['version'] = vers if vers else self.default_version - - def _get_topic(self, topic): - """Return the topic to use for a message.""" - return topic if topic else self.topic - - @staticmethod - def make_msg(method, **kwargs): - return {'method': method, 'args': kwargs} - - def call(self, context, msg, topic=None, version=None, timeout=None): - """rpc.call() a remote method. - - :param context: The request context - :param msg: The message to send, including the method and args. - :param topic: Override the topic for this message. - :param timeout: (Optional) A timeout to use when waiting for the - response. If no timeout is specified, a default timeout will be - used that is usually sufficient. - :param version: (Optional) Override the requested API version in this - message. - - :returns: The return value from the remote method. - """ - self._set_version(msg, version) - return rpc.call(context, self._get_topic(topic), msg, timeout) - - def multicall(self, context, msg, topic=None, version=None, timeout=None): - """rpc.multicall() a remote method. - - :param context: The request context - :param msg: The message to send, including the method and args. - :param topic: Override the topic for this message. - :param timeout: (Optional) A timeout to use when waiting for the - response. If no timeout is specified, a default timeout will be - used that is usually sufficient. - :param version: (Optional) Override the requested API version in this - message. - - :returns: An iterator that lets you process each of the returned values - from the remote method as they arrive. - """ - self._set_version(msg, version) - return rpc.multicall(context, self._get_topic(topic), msg, timeout) - - def cast(self, context, msg, topic=None, version=None): - """rpc.cast() a remote method. - - :param context: The request context - :param msg: The message to send, including the method and args. - :param topic: Override the topic for this message. - :param version: (Optional) Override the requested API version in this - message. - - :returns: None. rpc.cast() does not wait on any return value from the - remote method. - """ - self._set_version(msg, version) - rpc.cast(context, self._get_topic(topic), msg) - - def fanout_cast(self, context, msg, topic=None, version=None): - """rpc.fanout_cast() a remote method. - - :param context: The request context - :param msg: The message to send, including the method and args. - :param topic: Override the topic for this message. - :param version: (Optional) Override the requested API version in this - message. - - :returns: None. rpc.fanout_cast() does not wait on any return value - from the remote method. - """ - self._set_version(msg, version) - rpc.fanout_cast(context, self._get_topic(topic), msg) - - def cast_to_server(self, context, server_params, msg, topic=None, - version=None): - """rpc.cast_to_server() a remote method. - - :param context: The request context - :param server_params: Server parameters. See rpc.cast_to_server() for - details. - :param msg: The message to send, including the method and args. - :param topic: Override the topic for this message. - :param version: (Optional) Override the requested API version in this - message. - - :returns: None. rpc.cast_to_server() does not wait on any - return values. - """ - self._set_version(msg, version) - rpc.cast_to_server(context, server_params, self._get_topic(topic), msg) - - def fanout_cast_to_server(self, context, server_params, msg, topic=None, - version=None): - """rpc.fanout_cast_to_server() a remote method. - - :param context: The request context - :param server_params: Server parameters. See rpc.cast_to_server() for - details. - :param msg: The message to send, including the method and args. - :param topic: Override the topic for this message. - :param version: (Optional) Override the requested API version in this - message. - - :returns: None. rpc.fanout_cast_to_server() does not wait on any - return values. - """ - self._set_version(msg, version) - rpc.fanout_cast_to_server(context, server_params, - self._get_topic(topic), msg) diff --git a/bufunfa/openstack/common/rpc/service.py b/bufunfa/openstack/common/rpc/service.py deleted file mode 100644 index b553ce9..0000000 --- a/bufunfa/openstack/common/rpc/service.py +++ /dev/null @@ -1,70 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# Copyright 2011 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common import rpc -from bufunfa.openstack.common.rpc import dispatcher as rpc_dispatcher -from bufunfa.openstack.common import service - - -LOG = logging.getLogger(__name__) - - -class Service(service.Service): - """Service object for binaries running on hosts. - - A service enables rpc by listening to queues based on topic and host.""" - def __init__(self, host, topic, manager=None): - super(Service, self).__init__() - self.host = host - self.topic = topic - if manager is None: - self.manager = self - else: - self.manager = manager - - def start(self): - super(Service, self).start() - - self.conn = rpc.create_connection(new=True) - LOG.debug(_("Creating Consumer connection for Service %s") % - self.topic) - - dispatcher = rpc_dispatcher.RpcDispatcher([self.manager]) - - # Share this same connection for these Consumers - self.conn.create_consumer(self.topic, dispatcher, fanout=False) - - node_topic = '%s.%s' % (self.topic, self.host) - self.conn.create_consumer(node_topic, dispatcher, fanout=False) - - self.conn.create_consumer(self.topic, dispatcher, fanout=True) - - # Consume from all consumers in a thread - self.conn.consume_in_thread_group(self.tg) - - def stop(self): - # Try to shut the connection down, but if we get any sort of - # errors, go ahead and ignore them.. as we're shutting down anyway - try: - self.conn.close() - except Exception: - pass - super(Service, self).stop() diff --git a/bufunfa/openstack/common/service.py b/bufunfa/openstack/common/service.py deleted file mode 100644 index 0c50a14..0000000 --- a/bufunfa/openstack/common/service.py +++ /dev/null @@ -1,328 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Generic Node base class for all workers that run on hosts.""" - -import errno -import os -import random -import signal -import sys -import time - -import eventlet -import greenlet -import logging as std_logging - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import eventlet_backdoor -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common import threadgroup -from bufunfa.openstack.common.gettextutils import _ - -try: - from bufunfa.openstack.common import rpc -except ImportError: - rpc = None - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - - -class Launcher(object): - """Launch one or more services and wait for them to complete.""" - - def __init__(self): - """Initialize the service launcher. - - :returns: None - - """ - self._services = [] - eventlet_backdoor.initialize_if_enabled() - - @staticmethod - def run_service(service): - """Start and wait for a service to finish. - - :param service: service to run and wait for. - :returns: None - - """ - service.start() - service.wait() - - def launch_service(self, service): - """Load and start the given service. - - :param service: The service you would like to start. - :returns: None - - """ - gt = eventlet.spawn(self.run_service, service) - self._services.append(gt) - - def stop(self): - """Stop all services which are currently running. - - :returns: None - - """ - for service in self._services: - service.kill() - - def wait(self): - """Waits until all services have been stopped, and then returns. - - :returns: None - - """ - for service in self._services: - try: - service.wait() - except greenlet.GreenletExit: - pass - - -class SignalExit(SystemExit): - def __init__(self, signo, exccode=1): - super(SignalExit, self).__init__(exccode) - self.signo = signo - - -class ServiceLauncher(Launcher): - def _handle_signal(self, signo, frame): - # Allow the process to be killed again and die from natural causes - signal.signal(signal.SIGTERM, signal.SIG_DFL) - signal.signal(signal.SIGINT, signal.SIG_DFL) - - raise SignalExit(signo) - - def wait(self): - signal.signal(signal.SIGTERM, self._handle_signal) - signal.signal(signal.SIGINT, self._handle_signal) - - LOG.debug(_('Full set of CONF:')) - CONF.log_opt_values(LOG, std_logging.DEBUG) - - status = None - try: - super(ServiceLauncher, self).wait() - except SignalExit as exc: - signame = {signal.SIGTERM: 'SIGTERM', - signal.SIGINT: 'SIGINT'}[exc.signo] - LOG.info(_('Caught %s, exiting'), signame) - status = exc.code - except SystemExit as exc: - status = exc.code - finally: - self.stop() - if rpc: - rpc.cleanup() - return status - - -class ServiceWrapper(object): - def __init__(self, service, workers): - self.service = service - self.workers = workers - self.children = set() - self.forktimes = [] - - -class ProcessLauncher(object): - def __init__(self): - self.children = {} - self.sigcaught = None - self.running = True - rfd, self.writepipe = os.pipe() - self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') - - signal.signal(signal.SIGTERM, self._handle_signal) - signal.signal(signal.SIGINT, self._handle_signal) - - def _handle_signal(self, signo, frame): - self.sigcaught = signo - self.running = False - - # Allow the process to be killed again and die from natural causes - signal.signal(signal.SIGTERM, signal.SIG_DFL) - signal.signal(signal.SIGINT, signal.SIG_DFL) - - def _pipe_watcher(self): - # This will block until the write end is closed when the parent - # dies unexpectedly - self.readpipe.read() - - LOG.info(_('Parent process has died unexpectedly, exiting')) - - sys.exit(1) - - def _child_process(self, service): - # Setup child signal handlers differently - def _sigterm(*args): - signal.signal(signal.SIGTERM, signal.SIG_DFL) - raise SignalExit(signal.SIGTERM) - - signal.signal(signal.SIGTERM, _sigterm) - # Block SIGINT and let the parent send us a SIGTERM - signal.signal(signal.SIGINT, signal.SIG_IGN) - - # Reopen the eventlet hub to make sure we don't share an epoll - # fd with parent and/or siblings, which would be bad - eventlet.hubs.use_hub() - - # Close write to ensure only parent has it open - os.close(self.writepipe) - # Create greenthread to watch for parent to close pipe - eventlet.spawn(self._pipe_watcher) - - # Reseed random number generator - random.seed() - - launcher = Launcher() - launcher.run_service(service) - - def _start_child(self, wrap): - if len(wrap.forktimes) > wrap.workers: - # Limit ourselves to one process a second (over the period of - # number of workers * 1 second). This will allow workers to - # start up quickly but ensure we don't fork off children that - # die instantly too quickly. - if time.time() - wrap.forktimes[0] < wrap.workers: - LOG.info(_('Forking too fast, sleeping')) - time.sleep(1) - - wrap.forktimes.pop(0) - - wrap.forktimes.append(time.time()) - - pid = os.fork() - if pid == 0: - # NOTE(johannes): All exceptions are caught to ensure this - # doesn't fallback into the loop spawning children. It would - # be bad for a child to spawn more children. - status = 0 - try: - self._child_process(wrap.service) - except SignalExit as exc: - signame = {signal.SIGTERM: 'SIGTERM', - signal.SIGINT: 'SIGINT'}[exc.signo] - LOG.info(_('Caught %s, exiting'), signame) - status = exc.code - except SystemExit as exc: - status = exc.code - except BaseException: - LOG.exception(_('Unhandled exception')) - status = 2 - finally: - wrap.service.stop() - - os._exit(status) - - LOG.info(_('Started child %d'), pid) - - wrap.children.add(pid) - self.children[pid] = wrap - - return pid - - def launch_service(self, service, workers=1): - wrap = ServiceWrapper(service, workers) - - LOG.info(_('Starting %d workers'), wrap.workers) - while self.running and len(wrap.children) < wrap.workers: - self._start_child(wrap) - - def _wait_child(self): - try: - pid, status = os.wait() - except OSError as exc: - if exc.errno not in (errno.EINTR, errno.ECHILD): - raise - return None - - if os.WIFSIGNALED(status): - sig = os.WTERMSIG(status) - LOG.info(_('Child %(pid)d killed by signal %(sig)d'), locals()) - else: - code = os.WEXITSTATUS(status) - LOG.info(_('Child %(pid)d exited with status %(code)d'), locals()) - - if pid not in self.children: - LOG.warning(_('pid %d not in child list'), pid) - return None - - wrap = self.children.pop(pid) - wrap.children.remove(pid) - return wrap - - def wait(self): - """Loop waiting on children to die and respawning as necessary""" - while self.running: - wrap = self._wait_child() - if not wrap: - continue - - while self.running and len(wrap.children) < wrap.workers: - self._start_child(wrap) - - if self.sigcaught: - signame = {signal.SIGTERM: 'SIGTERM', - signal.SIGINT: 'SIGINT'}[self.sigcaught] - LOG.info(_('Caught %s, stopping children'), signame) - - for pid in self.children: - try: - os.kill(pid, signal.SIGTERM) - except OSError as exc: - if exc.errno != errno.ESRCH: - raise - - # Wait for children to die - if self.children: - LOG.info(_('Waiting on %d children to exit'), len(self.children)) - while self.children: - self._wait_child() - - -class Service(object): - """Service object for binaries running on hosts.""" - - def __init__(self): - self.tg = threadgroup.ThreadGroup('service') - - def start(self): - pass - - def stop(self): - self.tg.stop() - - def wait(self): - self.tg.wait() - - -def launch(service, workers=None): - if workers: - launcher = ProcessLauncher() - launcher.launch_service(service, workers=workers) - else: - launcher = ServiceLauncher() - launcher.launch_service(service) - return launcher diff --git a/bufunfa/openstack/common/setup.py b/bufunfa/openstack/common/setup.py deleted file mode 100644 index 83eef07..0000000 --- a/bufunfa/openstack/common/setup.py +++ /dev/null @@ -1,362 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Utilities with minimum-depends for use in setup.py -""" - -import datetime -import os -import re -import subprocess -import sys - -from setuptools.command import sdist - - -def parse_mailmap(mailmap='.mailmap'): - mapping = {} - if os.path.exists(mailmap): - with open(mailmap, 'r') as fp: - for l in fp: - l = l.strip() - if not l.startswith('#') and ' ' in l: - canonical_email, alias = [x for x in l.split(' ') - if x.startswith('<')] - mapping[alias] = canonical_email - return mapping - - -def canonicalize_emails(changelog, mapping): - """Takes in a string and an email alias mapping and replaces all - instances of the aliases in the string with their real email. - """ - for alias, email in mapping.iteritems(): - changelog = changelog.replace(alias, email) - return changelog - - -# Get requirements from the first file that exists -def get_reqs_from_files(requirements_files): - for requirements_file in requirements_files: - if os.path.exists(requirements_file): - with open(requirements_file, 'r') as fil: - return fil.read().split('\n') - return [] - - -def parse_requirements(requirements_files=['requirements.txt', - 'tools/pip-requires']): - requirements = [] - for line in get_reqs_from_files(requirements_files): - # For the requirements list, we need to inject only the portion - # after egg= so that distutils knows the package it's looking for - # such as: - # -e git://github.com/openstack/nova/master#egg=nova - if re.match(r'\s*-e\s+', line): - requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', - line)) - # such as: - # http://github.com/openstack/nova/zipball/master#egg=nova - elif re.match(r'\s*https?:', line): - requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', - line)) - # -f lines are for index locations, and don't get used here - elif re.match(r'\s*-f\s+', line): - pass - # argparse is part of the standard library starting with 2.7 - # adding it to the requirements list screws distro installs - elif line == 'argparse' and sys.version_info >= (2, 7): - pass - else: - requirements.append(line) - - return requirements - - -def parse_dependency_links(requirements_files=['requirements.txt', - 'tools/pip-requires']): - dependency_links = [] - # dependency_links inject alternate locations to find packages listed - # in requirements - for line in get_reqs_from_files(requirements_files): - # skip comments and blank lines - if re.match(r'(\s*#)|(\s*$)', line): - continue - # lines with -e or -f need the whole line, minus the flag - if re.match(r'\s*-[ef]\s+', line): - dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line)) - # lines that are only urls can go in unmolested - elif re.match(r'\s*https?:', line): - dependency_links.append(line) - return dependency_links - - -def write_requirements(): - venv = os.environ.get('VIRTUAL_ENV', None) - if venv is not None: - with open("requirements.txt", "w") as req_file: - output = subprocess.Popen(["pip", "-E", venv, "freeze", "-l"], - stdout=subprocess.PIPE) - requirements = output.communicate()[0].strip() - req_file.write(requirements) - - -def _run_shell_command(cmd): - output = subprocess.Popen(["/bin/sh", "-c", cmd], - stdout=subprocess.PIPE) - out = output.communicate() - if len(out) == 0: - return None - if len(out[0].strip()) == 0: - return None - return out[0].strip() - - -def _get_git_next_version_suffix(branch_name): - datestamp = datetime.datetime.now().strftime('%Y%m%d') - if branch_name == 'milestone-proposed': - revno_prefix = "r" - else: - revno_prefix = "" - _run_shell_command("git fetch origin +refs/meta/*:refs/remotes/meta/*") - milestone_cmd = "git show meta/openstack/release:%s" % branch_name - milestonever = _run_shell_command(milestone_cmd) - if milestonever: - first_half = "%s~%s" % (milestonever, datestamp) - else: - first_half = datestamp - - post_version = _get_git_post_version() - # post version should look like: - # 0.1.1.4.gcc9e28a - # where the bit after the last . is the short sha, and the bit between - # the last and second to last is the revno count - (revno, sha) = post_version.split(".")[-2:] - second_half = "%s%s.%s" % (revno_prefix, revno, sha) - return ".".join((first_half, second_half)) - - -def _get_git_current_tag(): - return _run_shell_command("git tag --contains HEAD") - - -def _get_git_tag_info(): - return _run_shell_command("git describe --tags") - - -def _get_git_post_version(): - current_tag = _get_git_current_tag() - if current_tag is not None: - return current_tag - else: - tag_info = _get_git_tag_info() - if tag_info is None: - base_version = "0.0" - cmd = "git --no-pager log --oneline" - out = _run_shell_command(cmd) - revno = len(out.split("\n")) - sha = _run_shell_command("git describe --always") - else: - tag_infos = tag_info.split("-") - base_version = "-".join(tag_infos[:-2]) - (revno, sha) = tag_infos[-2:] - return "%s.%s.%s" % (base_version, revno, sha) - - -def write_git_changelog(): - """Write a changelog based on the git changelog.""" - new_changelog = 'ChangeLog' - if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'): - if os.path.isdir('.git'): - git_log_cmd = 'git log --stat' - changelog = _run_shell_command(git_log_cmd) - mailmap = parse_mailmap() - with open(new_changelog, "w") as changelog_file: - changelog_file.write(canonicalize_emails(changelog, mailmap)) - else: - open(new_changelog, 'w').close() - - -def generate_authors(): - """Create AUTHORS file using git commits.""" - jenkins_email = 'jenkins@review.(openstack|stackforge).org' - old_authors = 'AUTHORS.in' - new_authors = 'AUTHORS' - if not os.getenv('SKIP_GENERATE_AUTHORS'): - if os.path.isdir('.git'): - # don't include jenkins email address in AUTHORS file - git_log_cmd = ("git log --format='%aN <%aE>' | sort -u | " - "egrep -v '" + jenkins_email + "'") - changelog = _run_shell_command(git_log_cmd) - mailmap = parse_mailmap() - with open(new_authors, 'w') as new_authors_fh: - new_authors_fh.write(canonicalize_emails(changelog, mailmap)) - if os.path.exists(old_authors): - with open(old_authors, "r") as old_authors_fh: - new_authors_fh.write('\n' + old_authors_fh.read()) - else: - open(new_authors, 'w').close() - - -_rst_template = """%(heading)s -%(underline)s - -.. automodule:: %(module)s - :members: - :undoc-members: - :show-inheritance: -""" - - -def read_versioninfo(project): - """Read the versioninfo file. If it doesn't exist, we're in a github - zipball, and there's really no way to know what version we really - are, but that should be ok, because the utility of that should be - just about nil if this code path is in use in the first place.""" - versioninfo_path = os.path.join(project, 'versioninfo') - if os.path.exists(versioninfo_path): - with open(versioninfo_path, 'r') as vinfo: - version = vinfo.read().strip() - else: - version = "0.0.0" - return version - - -def write_versioninfo(project, version): - """Write a simple file containing the version of the package.""" - with open(os.path.join(project, 'versioninfo'), 'w') as fil: - fil.write("%s\n" % version) - - -def get_cmdclass(): - """Return dict of commands to run from setup.py.""" - - cmdclass = dict() - - def _find_modules(arg, dirname, files): - for filename in files: - if filename.endswith('.py') and filename != '__init__.py': - arg["%s.%s" % (dirname.replace('/', '.'), - filename[:-3])] = True - - class LocalSDist(sdist.sdist): - """Builds the ChangeLog and Authors files from VC first.""" - - def run(self): - write_git_changelog() - generate_authors() - # sdist.sdist is an old style class, can't use super() - sdist.sdist.run(self) - - cmdclass['sdist'] = LocalSDist - - # If Sphinx is installed on the box running setup.py, - # enable setup.py to build the documentation, otherwise, - # just ignore it - try: - from sphinx.setup_command import BuildDoc - - class LocalBuildDoc(BuildDoc): - def generate_autoindex(self): - print "**Autodocumenting from %s" % os.path.abspath(os.curdir) - modules = {} - option_dict = self.distribution.get_option_dict('build_sphinx') - source_dir = os.path.join(option_dict['source_dir'][1], 'api') - if not os.path.exists(source_dir): - os.makedirs(source_dir) - for pkg in self.distribution.packages: - if '.' not in pkg: - os.path.walk(pkg, _find_modules, modules) - module_list = modules.keys() - module_list.sort() - autoindex_filename = os.path.join(source_dir, 'autoindex.rst') - with open(autoindex_filename, 'w') as autoindex: - autoindex.write(""".. toctree:: - :maxdepth: 1 - -""") - for module in module_list: - output_filename = os.path.join(source_dir, - "%s.rst" % module) - heading = "The :mod:`%s` Module" % module - underline = "=" * len(heading) - values = dict(module=module, heading=heading, - underline=underline) - - print "Generating %s" % output_filename - with open(output_filename, 'w') as output_file: - output_file.write(_rst_template % values) - autoindex.write(" %s.rst\n" % module) - - def run(self): - if not os.getenv('SPHINX_DEBUG'): - self.generate_autoindex() - - for builder in ['html', 'man']: - self.builder = builder - self.finalize_options() - self.project = self.distribution.get_name() - self.version = self.distribution.get_version() - self.release = self.distribution.get_version() - BuildDoc.run(self) - cmdclass['build_sphinx'] = LocalBuildDoc - except ImportError: - pass - - return cmdclass - - -def get_git_branchname(): - for branch in _run_shell_command("git branch --color=never").split("\n"): - if branch.startswith('*'): - _branch_name = branch.split()[1].strip() - if _branch_name == "(no": - _branch_name = "no-branch" - return _branch_name - - -def get_pre_version(projectname, base_version): - """Return a version which is leading up to a version that will - be released in the future.""" - if os.path.isdir('.git'): - current_tag = _get_git_current_tag() - if current_tag is not None: - version = current_tag - else: - branch_name = os.getenv('BRANCHNAME', - os.getenv('GERRIT_REFNAME', - get_git_branchname())) - version_suffix = _get_git_next_version_suffix(branch_name) - version = "%s~%s" % (base_version, version_suffix) - write_versioninfo(projectname, version) - return version - else: - version = read_versioninfo(projectname) - return version - - -def get_post_version(projectname): - """Return a version which is equal to the tag that's on the current - revision if there is one, or tag plus number of additional revisions - if the current revision has no tag.""" - - if os.path.isdir('.git'): - version = _get_git_post_version() - write_versioninfo(projectname, version) - return version - return read_versioninfo(projectname) diff --git a/bufunfa/openstack/common/threadgroup.py b/bufunfa/openstack/common/threadgroup.py deleted file mode 100644 index 87c0eaa..0000000 --- a/bufunfa/openstack/common/threadgroup.py +++ /dev/null @@ -1,118 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import os -import sys - -from eventlet import event -from eventlet import greenthread -from eventlet import greenpool - -from bufunfa.openstack.common import loopingcall -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import log as logging - - -LOG = logging.getLogger(__name__) - - -def _thread_done(gt, *args, **kwargs): - args[0].thread_done(args[1]) - - -class Thread(object): - """ - Wrapper around a greenthread, that holds a reference to - the ThreadGroup. The Thread will notify the ThreadGroup - when it has done so it can be removed from the threads - list. - """ - def __init__(self, name, thread, group): - self.name = name - self.thread = thread - self.thread.link(_thread_done, group, self) - - def stop(self): - self.thread.cancel() - - def wait(self): - return self.thread.wait() - - -class ThreadGroup(): - """ - The point of this class is to: - - keep track of timers and greenthreads (making it easier to stop them - when need be). - - provide an easy API to add timers. - """ - def __init__(self, name, thread_pool_size=10): - self.name = name - self.pool = greenpool.GreenPool(thread_pool_size) - self.threads = [] - self.timers = [] - - def add_timer(self, interval, callback, initial_delay=None, - *args, **kwargs): - pulse = loopingcall.LoopingCall(callback, *args, **kwargs) - pulse.start(interval=interval, - initial_delay=initial_delay) - self.timers.append(pulse) - - def add_thread(self, callback, *args, **kwargs): - gt = self.pool.spawn(callback, *args, **kwargs) - th = Thread(callback.__name__, gt, self) - self.threads.append(th) - - def thread_done(self, thread): - try: - thread.wait() - except Exception as ex: - LOG.exception(ex) - finally: - self.threads.remove(thread) - - def stop(self): - current = greenthread.getcurrent() - for x in self.threads: - if x is current: - # don't kill the current thread. - continue - try: - x.stop() - except Exception as ex: - LOG.exception(ex) - - for x in self.timers: - try: - x.stop() - except Exception as ex: - LOG.exception(ex) - self.timers = [] - - def wait(self): - for x in self.timers: - try: - x.wait() - except Exception as ex: - LOG.exception(ex) - current = greenthread.getcurrent() - for x in self.threads: - if x is current: - continue - try: - x.wait() - except Exception as ex: - LOG.exception(ex) diff --git a/bufunfa/openstack/common/timeutils.py b/bufunfa/openstack/common/timeutils.py deleted file mode 100644 index 8600439..0000000 --- a/bufunfa/openstack/common/timeutils.py +++ /dev/null @@ -1,137 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Time related utilities and helper functions. -""" - -import calendar -import datetime - -import iso8601 - - -TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" -PERFECT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" - - -def isotime(at=None): - """Stringify time in ISO 8601 format""" - if not at: - at = utcnow() - str = at.strftime(TIME_FORMAT) - tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' - str += ('Z' if tz == 'UTC' else tz) - return str - - -def parse_isotime(timestr): - """Parse time from ISO 8601 format""" - try: - return iso8601.parse_date(timestr) - except iso8601.ParseError as e: - raise ValueError(e.message) - except TypeError as e: - raise ValueError(e.message) - - -def strtime(at=None, fmt=PERFECT_TIME_FORMAT): - """Returns formatted utcnow.""" - if not at: - at = utcnow() - return at.strftime(fmt) - - -def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): - """Turn a formatted time back into a datetime.""" - return datetime.datetime.strptime(timestr, fmt) - - -def normalize_time(timestamp): - """Normalize time in arbitrary timezone to UTC naive object""" - offset = timestamp.utcoffset() - if offset is None: - return timestamp - return timestamp.replace(tzinfo=None) - offset - - -def is_older_than(before, seconds): - """Return True if before is older than seconds.""" - return utcnow() - before > datetime.timedelta(seconds=seconds) - - -def is_newer_than(after, seconds): - """Return True if after is newer than seconds.""" - return after - utcnow() > datetime.timedelta(seconds=seconds) - - -def utcnow_ts(): - """Timestamp version of our utcnow function.""" - return calendar.timegm(utcnow().timetuple()) - - -def utcnow(): - """Overridable version of utils.utcnow.""" - if utcnow.override_time: - return utcnow.override_time - return datetime.datetime.utcnow() - - -utcnow.override_time = None - - -def set_time_override(override_time=datetime.datetime.utcnow()): - """Override utils.utcnow to return a constant time.""" - utcnow.override_time = override_time - - -def advance_time_delta(timedelta): - """Advance overridden time using a datetime.timedelta.""" - assert(not utcnow.override_time is None) - utcnow.override_time += timedelta - - -def advance_time_seconds(seconds): - """Advance overridden time by seconds.""" - advance_time_delta(datetime.timedelta(0, seconds)) - - -def clear_time_override(): - """Remove the overridden time.""" - utcnow.override_time = None - - -def marshall_now(now=None): - """Make an rpc-safe datetime with microseconds. - - Note: tzinfo is stripped, but not required for relative times.""" - if not now: - now = utcnow() - return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, - minute=now.minute, second=now.second, - microsecond=now.microsecond) - - -def unmarshall_time(tyme): - """Unmarshall a datetime dict.""" - return datetime.datetime(day=tyme['day'], - month=tyme['month'], - year=tyme['year'], - hour=tyme['hour'], - minute=tyme['minute'], - second=tyme['second'], - microsecond=tyme['microsecond']) diff --git a/bufunfa/openstack/common/utils.py b/bufunfa/openstack/common/utils.py deleted file mode 100644 index 9e56cf9..0000000 --- a/bufunfa/openstack/common/utils.py +++ /dev/null @@ -1,64 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -System-level utilities and helper functions. -""" - -import logging -import random -import shlex - -from eventlet.green import subprocess -from eventlet import greenthread - -from bufunfa.openstack.common import exception -from bufunfa.openstack.common.gettextutils import _ - - -LOG = logging.getLogger(__name__) - - -def int_from_bool_as_string(subject): - """ - Interpret a string as a boolean and return either 1 or 0. - - Any string value in: - ('True', 'true', 'On', 'on', '1') - is interpreted as a boolean True. - - Useful for JSON-decoded stuff and config file parsing - """ - return bool_from_string(subject) and 1 or 0 - - -def bool_from_string(subject): - """ - Interpret a string as a boolean. - - Any string value in: - ('True', 'true', 'On', 'on', 'Yes', 'yes', '1') - is interpreted as a boolean True. - - Useful for JSON-decoded stuff and config file parsing - """ - if isinstance(subject, bool): - return subject - if isinstance(subject, basestring): - if subject.strip().lower() in ('true', 'on', 'yes', '1'): - return True - return False diff --git a/bufunfa/openstack/common/version.py b/bufunfa/openstack/common/version.py deleted file mode 100644 index a19e422..0000000 --- a/bufunfa/openstack/common/version.py +++ /dev/null @@ -1,148 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 OpenStack LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Utilities for consuming the auto-generated versioninfo files. -""" - -import datetime -import pkg_resources - -import setup - - -class _deferred_version_string(object): - """Internal helper class which provides delayed version calculation.""" - def __init__(self, version_info, prefix): - self.version_info = version_info - self.prefix = prefix - - def __str__(self): - return "%s%s" % (self.prefix, self.version_info.version_string()) - - def __repr__(self): - return "%s%s" % (self.prefix, self.version_info.version_string()) - - -class VersionInfo(object): - - def __init__(self, package, python_package=None, pre_version=None): - """Object that understands versioning for a package - :param package: name of the top level python namespace. For glance, - this would be "glance" for python-glanceclient, it - would be "glanceclient" - :param python_package: optional name of the project name. For - glance this can be left unset. For - python-glanceclient, this would be - "python-glanceclient" - :param pre_version: optional version that the project is working to - """ - self.package = package - if python_package is None: - self.python_package = package - else: - self.python_package = python_package - self.pre_version = pre_version - self.version = None - - def _generate_version(self): - """Defer to the openstack.common.setup routines for making a - version from git.""" - if self.pre_version is None: - return setup.get_post_version(self.python_package) - else: - return setup.get_pre_version(self.python_package, self.pre_version) - - def _newer_version(self, pending_version): - """Check to see if we're working with a stale version or not. - We expect a version string that either looks like: - 2012.2~f3~20120708.10.4426392 - which is an unreleased version of a pre-version, or: - 0.1.1.4.gcc9e28a - which is an unreleased version of a post-version, or: - 0.1.1 - Which is a release and which should match tag. - For now, if we have a date-embedded version, check to see if it's - old, and if so re-generate. Otherwise, just deal with it. - """ - try: - version_date = int(self.version.split("~")[-1].split('.')[0]) - if version_date < int(datetime.date.today().strftime('%Y%m%d')): - return self._generate_version() - else: - return pending_version - except Exception: - return pending_version - - def version_string_with_vcs(self, always=False): - """Return the full version of the package including suffixes indicating - VCS status. - - For instance, if we are working towards the 2012.2 release, - canonical_version_string should return 2012.2 if this is a final - release, or else something like 2012.2~f1~20120705.20 if it's not. - - :param always: if true, skip all version caching - """ - if always: - self.version = self._generate_version() - - if self.version is None: - - requirement = pkg_resources.Requirement.parse(self.python_package) - versioninfo = "%s/versioninfo" % self.package - try: - raw_version = pkg_resources.resource_string(requirement, - versioninfo) - self.version = self._newer_version(raw_version.strip()) - except (IOError, pkg_resources.DistributionNotFound): - self.version = self._generate_version() - - return self.version - - def canonical_version_string(self, always=False): - """Return the simple version of the package excluding any suffixes. - - For instance, if we are working towards the 2012.2 release, - canonical_version_string should return 2012.2 in all cases. - - :param always: if true, skip all version caching - """ - return self.version_string_with_vcs(always).split('~')[0] - - def version_string(self, always=False): - """Return the base version of the package. - - For instance, if we are working towards the 2012.2 release, - version_string should return 2012.2 if this is a final release, or - 2012.2-dev if it is not. - - :param always: if true, skip all version caching - """ - version_parts = self.version_string_with_vcs(always).split('~') - if len(version_parts) == 1: - return version_parts[0] - else: - return '%s-dev' % (version_parts[0],) - - def deferred_version_string(self, prefix=""): - """Generate an object which will expand in a string context to - the results of version_string(). We do this so that don't - call into pkg_resources every time we start up a program when - passing version information into the CONF constructor, but - rather only do the calculation when and if a version is requested - """ - return _deferred_version_string(self, prefix) diff --git a/bufunfa/openstack/common/wsgi.py b/bufunfa/openstack/common/wsgi.py deleted file mode 100644 index 4bf62a7..0000000 --- a/bufunfa/openstack/common/wsgi.py +++ /dev/null @@ -1,728 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Utility methods for working with WSGI servers.""" - -import datetime -import eventlet -import eventlet.wsgi - -eventlet.patcher.monkey_patch(all=False, socket=True) - -import routes -import routes.middleware -import sys -import webob.dec -import webob.exc -from xml.dom import minidom -from xml.parsers import expat - -from bufunfa.openstack.common import exception -from bufunfa.openstack.common.gettextutils import _ -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common import jsonutils -from bufunfa.openstack.common import service - - -LOG = logging.getLogger(__name__) - - -def run_server(application, port): - """Run a WSGI server with the given application.""" - sock = eventlet.listen(('0.0.0.0', port)) - eventlet.wsgi.server(sock, application) - - -class Service(service.Service): - """ - Provides a Service API for wsgi servers. - - This gives us the ability to launch wsgi servers with the - Launcher classes in service.py. - """ - - def __init__(self, threads=1000): - super(Service, self).__init__() - self.pool = eventlet.GreenPool(threads) - - def start(self, application, port, host='0.0.0.0', backlog=128): - """Start serving this service using the provided server instance. - - :returns: None - - """ - super(Service, self).start() - socket = eventlet.listen((host, port), backlog=backlog) - self.pool.spawn_n(self._run, application, socket) - - def stop(self): - """Stop serving this API. - - :returns: None - - """ - super(Service, self).stop() - - def wait(self): - """Wait until all servers have completed running.""" - super(Service, self).wait() - try: - self.pool.waitall() - except KeyboardInterrupt: - pass - - def _run(self, application, socket): - """Start a WSGI server in a new green thread.""" - logger = logging.getLogger('eventlet.wsgi.server') - eventlet.wsgi.server(socket, application, custom_pool=self.pool, - log=logging.WritableLogger(logger)) - - -class Middleware(object): - """ - Base WSGI middleware wrapper. These classes require an application to be - initialized that will be called next. By default the middleware will - simply call its wrapped app, or you can override __call__ to customize its - behavior. - """ - - def __init__(self, application): - self.application = application - - def process_request(self, req): - """ - Called on each request. - - If this returns None, the next application down the stack will be - executed. If it returns a response then that response will be returned - and execution will stop here. - """ - return None - - def process_response(self, response): - """Do whatever you'd like to the response.""" - return response - - @webob.dec.wsgify - def __call__(self, req): - response = self.process_request(req) - if response: - return response - response = req.get_response(self.application) - return self.process_response(response) - - -class Debug(Middleware): - """ - Helper class that can be inserted into any WSGI application chain - to get information about the request and response. - """ - - @webob.dec.wsgify - def __call__(self, req): - print ("*" * 40) + " REQUEST ENVIRON" - for key, value in req.environ.items(): - print key, "=", value - print - resp = req.get_response(self.application) - - print ("*" * 40) + " RESPONSE HEADERS" - for (key, value) in resp.headers.iteritems(): - print key, "=", value - print - - resp.app_iter = self.print_generator(resp.app_iter) - - return resp - - @staticmethod - def print_generator(app_iter): - """ - Iterator that prints the contents of a wrapper string iterator - when iterated. - """ - print ("*" * 40) + " BODY" - for part in app_iter: - sys.stdout.write(part) - sys.stdout.flush() - yield part - print - - -class Router(object): - - """ - WSGI middleware that maps incoming requests to WSGI apps. - """ - - def __init__(self, mapper): - """ - Create a router for the given routes.Mapper. - - Each route in `mapper` must specify a 'controller', which is a - WSGI app to call. You'll probably want to specify an 'action' as - well and have your controller be a wsgi.Controller, who will route - the request to the action method. - - Examples: - mapper = routes.Mapper() - sc = ServerController() - - # Explicit mapping of one route to a controller+action - mapper.connect(None, "/svrlist", controller=sc, action="list") - - # Actions are all implicitly defined - mapper.resource("server", "servers", controller=sc) - - # Pointing to an arbitrary WSGI app. You can specify the - # {path_info:.*} parameter so the target app can be handed just that - # section of the URL. - mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp()) - """ - self.map = mapper - self._router = routes.middleware.RoutesMiddleware(self._dispatch, - self.map) - - @webob.dec.wsgify - def __call__(self, req): - """ - Route the incoming request to a controller based on self.map. - If no match, return a 404. - """ - return self._router - - @staticmethod - @webob.dec.wsgify - def _dispatch(req): - """ - Called by self._router after matching the incoming request to a route - and putting the information into req.environ. Either returns 404 - or the routed WSGI app's response. - """ - match = req.environ['wsgiorg.routing_args'][1] - if not match: - return webob.exc.HTTPNotFound() - app = match['controller'] - return app - - -class Request(webob.Request): - """Add some Openstack API-specific logic to the base webob.Request.""" - - default_request_content_types = ('application/json', 'application/xml') - default_accept_types = ('application/json', 'application/xml') - default_accept_type = 'application/json' - - def best_match_content_type(self, supported_content_types=None): - """Determine the requested response content-type. - - Based on the query extension then the Accept header. - Defaults to default_accept_type if we don't find a preference - - """ - supported_content_types = (supported_content_types or - self.default_accept_types) - - parts = self.path.rsplit('.', 1) - if len(parts) > 1: - ctype = 'application/{0}'.format(parts[1]) - if ctype in supported_content_types: - return ctype - - bm = self.accept.best_match(supported_content_types) - return bm or self.default_accept_type - - def get_content_type(self, allowed_content_types=None): - """Determine content type of the request body. - - Does not do any body introspection, only checks header - - """ - if not "Content-Type" in self.headers: - return None - - content_type = self.content_type - allowed_content_types = (allowed_content_types or - self.default_request_content_types) - - if content_type not in allowed_content_types: - raise exception.InvalidContentType(content_type=content_type) - return content_type - - -class Resource(object): - """ - WSGI app that handles (de)serialization and controller dispatch. - - Reads routing information supplied by RoutesMiddleware and calls - the requested action method upon its deserializer, controller, - and serializer. Those three objects may implement any of the basic - controller action methods (create, update, show, index, delete) - along with any that may be specified in the api router. A 'default' - method may also be implemented to be used in place of any - non-implemented actions. Deserializer methods must accept a request - argument and return a dictionary. Controller methods must accept a - request argument. Additionally, they must also accept keyword - arguments that represent the keys returned by the Deserializer. They - may raise a webob.exc exception or return a dict, which will be - serialized by requested content type. - """ - def __init__(self, controller, deserializer=None, serializer=None): - """ - :param controller: object that implement methods created by routes lib - :param deserializer: object that supports webob request deserialization - through controller-like actions - :param serializer: object that supports webob response serialization - through controller-like actions - """ - self.controller = controller - self.serializer = serializer or ResponseSerializer() - self.deserializer = deserializer or RequestDeserializer() - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, request): - """WSGI method that controls (de)serialization and method dispatch.""" - - try: - action, action_args, accept = self.deserialize_request(request) - except exception.InvalidContentType: - msg = _("Unsupported Content-Type") - return webob.exc.HTTPUnsupportedMediaType(explanation=msg) - except exception.MalformedRequestBody: - msg = _("Malformed request body") - return webob.exc.HTTPBadRequest(explanation=msg) - - action_result = self.execute_action(action, request, **action_args) - try: - return self.serialize_response(action, action_result, accept) - # return unserializable result (typically a webob exc) - except Exception: - return action_result - - def deserialize_request(self, request): - return self.deserializer.deserialize(request) - - def serialize_response(self, action, action_result, accept): - return self.serializer.serialize(action_result, accept, action) - - def execute_action(self, action, request, **action_args): - return self.dispatch(self.controller, action, request, **action_args) - - def dispatch(self, obj, action, *args, **kwargs): - """Find action-specific method on self and call it.""" - try: - method = getattr(obj, action) - except AttributeError: - method = getattr(obj, 'default') - - return method(*args, **kwargs) - - def get_action_args(self, request_environment): - """Parse dictionary created by routes library.""" - try: - args = request_environment['wsgiorg.routing_args'][1].copy() - except Exception: - return {} - - try: - del args['controller'] - except KeyError: - pass - - try: - del args['format'] - except KeyError: - pass - - return args - - -class ActionDispatcher(object): - """Maps method name to local methods through action name.""" - - def dispatch(self, *args, **kwargs): - """Find and call local method.""" - action = kwargs.pop('action', 'default') - action_method = getattr(self, str(action), self.default) - return action_method(*args, **kwargs) - - def default(self, data): - raise NotImplementedError() - - -class DictSerializer(ActionDispatcher): - """Default request body serialization""" - - def serialize(self, data, action='default'): - return self.dispatch(data, action=action) - - def default(self, data): - return "" - - -class JSONDictSerializer(DictSerializer): - """Default JSON request body serialization""" - - def default(self, data): - def sanitizer(obj): - if isinstance(obj, datetime.datetime): - _dtime = obj - datetime.timedelta(microseconds=obj.microsecond) - return _dtime.isoformat() - return obj - return jsonutils.dumps(data, default=sanitizer) - - -class XMLDictSerializer(DictSerializer): - - def __init__(self, metadata=None, xmlns=None): - """ - :param metadata: information needed to deserialize xml into - a dictionary. - :param xmlns: XML namespace to include with serialized xml - """ - super(XMLDictSerializer, self).__init__() - self.metadata = metadata or {} - self.xmlns = xmlns - - def default(self, data): - # We expect data to contain a single key which is the XML root. - root_key = data.keys()[0] - doc = minidom.Document() - node = self._to_xml_node(doc, self.metadata, root_key, data[root_key]) - - return self.to_xml_string(node) - - def to_xml_string(self, node, has_atom=False): - self._add_xmlns(node, has_atom) - return node.toprettyxml(indent=' ', encoding='UTF-8') - - #NOTE (ameade): the has_atom should be removed after all of the - # xml serializers and view builders have been updated to the current - # spec that required all responses include the xmlns:atom, the has_atom - # flag is to prevent current tests from breaking - def _add_xmlns(self, node, has_atom=False): - if self.xmlns is not None: - node.setAttribute('xmlns', self.xmlns) - if has_atom: - node.setAttribute('xmlns:atom', "http://www.w3.org/2005/Atom") - - def _to_xml_node(self, doc, metadata, nodename, data): - """Recursive method to convert data members to XML nodes.""" - result = doc.createElement(nodename) - - # Set the xml namespace if one is specified - # TODO(justinsb): We could also use prefixes on the keys - xmlns = metadata.get('xmlns', None) - if xmlns: - result.setAttribute('xmlns', xmlns) - - #TODO(bcwaldon): accomplish this without a type-check - if type(data) is list: - collections = metadata.get('list_collections', {}) - if nodename in collections: - metadata = collections[nodename] - for item in data: - node = doc.createElement(metadata['item_name']) - node.setAttribute(metadata['item_key'], str(item)) - result.appendChild(node) - return result - singular = metadata.get('plurals', {}).get(nodename, None) - if singular is None: - if nodename.endswith('s'): - singular = nodename[:-1] - else: - singular = 'item' - for item in data: - node = self._to_xml_node(doc, metadata, singular, item) - result.appendChild(node) - #TODO(bcwaldon): accomplish this without a type-check - elif type(data) is dict: - collections = metadata.get('dict_collections', {}) - if nodename in collections: - metadata = collections[nodename] - for k, v in data.items(): - node = doc.createElement(metadata['item_name']) - node.setAttribute(metadata['item_key'], str(k)) - text = doc.createTextNode(str(v)) - node.appendChild(text) - result.appendChild(node) - return result - attrs = metadata.get('attributes', {}).get(nodename, {}) - for k, v in data.items(): - if k in attrs: - result.setAttribute(k, str(v)) - else: - node = self._to_xml_node(doc, metadata, k, v) - result.appendChild(node) - else: - # Type is atom - node = doc.createTextNode(str(data)) - result.appendChild(node) - return result - - def _create_link_nodes(self, xml_doc, links): - link_nodes = [] - for link in links: - link_node = xml_doc.createElement('atom:link') - link_node.setAttribute('rel', link['rel']) - link_node.setAttribute('href', link['href']) - if 'type' in link: - link_node.setAttribute('type', link['type']) - link_nodes.append(link_node) - return link_nodes - - -class ResponseHeadersSerializer(ActionDispatcher): - """Default response headers serialization""" - - def serialize(self, response, data, action): - self.dispatch(response, data, action=action) - - def default(self, response, data): - response.status_int = 200 - - -class ResponseSerializer(object): - """Encode the necessary pieces into a response object""" - - def __init__(self, body_serializers=None, headers_serializer=None): - self.body_serializers = { - 'application/xml': XMLDictSerializer(), - 'application/json': JSONDictSerializer(), - } - self.body_serializers.update(body_serializers or {}) - - self.headers_serializer = (headers_serializer or - ResponseHeadersSerializer()) - - def serialize(self, response_data, content_type, action='default'): - """Serialize a dict into a string and wrap in a wsgi.Request object. - - :param response_data: dict produced by the Controller - :param content_type: expected mimetype of serialized response body - - """ - response = webob.Response() - self.serialize_headers(response, response_data, action) - self.serialize_body(response, response_data, content_type, action) - return response - - def serialize_headers(self, response, data, action): - self.headers_serializer.serialize(response, data, action) - - def serialize_body(self, response, data, content_type, action): - response.headers['Content-Type'] = content_type - if data is not None: - serializer = self.get_body_serializer(content_type) - response.body = serializer.serialize(data, action) - - def get_body_serializer(self, content_type): - try: - return self.body_serializers[content_type] - except (KeyError, TypeError): - raise exception.InvalidContentType(content_type=content_type) - - -class RequestHeadersDeserializer(ActionDispatcher): - """Default request headers deserializer""" - - def deserialize(self, request, action): - return self.dispatch(request, action=action) - - def default(self, request): - return {} - - -class RequestDeserializer(object): - """Break up a Request object into more useful pieces.""" - - def __init__(self, body_deserializers=None, headers_deserializer=None, - supported_content_types=None): - - self.supported_content_types = supported_content_types - - self.body_deserializers = { - 'application/xml': XMLDeserializer(), - 'application/json': JSONDeserializer(), - } - self.body_deserializers.update(body_deserializers or {}) - - self.headers_deserializer = (headers_deserializer or - RequestHeadersDeserializer()) - - def deserialize(self, request): - """Extract necessary pieces of the request. - - :param request: Request object - :returns tuple of expected controller action name, dictionary of - keyword arguments to pass to the controller, the expected - content type of the response - - """ - action_args = self.get_action_args(request.environ) - action = action_args.pop('action', None) - - action_args.update(self.deserialize_headers(request, action)) - action_args.update(self.deserialize_body(request, action)) - - accept = self.get_expected_content_type(request) - - return (action, action_args, accept) - - def deserialize_headers(self, request, action): - return self.headers_deserializer.deserialize(request, action) - - def deserialize_body(self, request, action): - if not len(request.body) > 0: - LOG.debug(_("Empty body provided in request")) - return {} - - try: - content_type = request.get_content_type() - except exception.InvalidContentType: - LOG.debug(_("Unrecognized Content-Type provided in request")) - raise - - if content_type is None: - LOG.debug(_("No Content-Type provided in request")) - return {} - - try: - deserializer = self.get_body_deserializer(content_type) - except exception.InvalidContentType: - LOG.debug(_("Unable to deserialize body as provided Content-Type")) - raise - - return deserializer.deserialize(request.body, action) - - def get_body_deserializer(self, content_type): - try: - return self.body_deserializers[content_type] - except (KeyError, TypeError): - raise exception.InvalidContentType(content_type=content_type) - - def get_expected_content_type(self, request): - return request.best_match_content_type(self.supported_content_types) - - def get_action_args(self, request_environment): - """Parse dictionary created by routes library.""" - try: - args = request_environment['wsgiorg.routing_args'][1].copy() - except Exception: - return {} - - try: - del args['controller'] - except KeyError: - pass - - try: - del args['format'] - except KeyError: - pass - - return args - - -class TextDeserializer(ActionDispatcher): - """Default request body deserialization""" - - def deserialize(self, datastring, action='default'): - return self.dispatch(datastring, action=action) - - def default(self, datastring): - return {} - - -class JSONDeserializer(TextDeserializer): - - def _from_json(self, datastring): - try: - return jsonutils.loads(datastring) - except ValueError: - msg = _("cannot understand JSON") - raise exception.MalformedRequestBody(reason=msg) - - def default(self, datastring): - return {'body': self._from_json(datastring)} - - -class XMLDeserializer(TextDeserializer): - - def __init__(self, metadata=None): - """ - :param metadata: information needed to deserialize xml into - a dictionary. - """ - super(XMLDeserializer, self).__init__() - self.metadata = metadata or {} - - def _from_xml(self, datastring): - plurals = set(self.metadata.get('plurals', {})) - - try: - node = minidom.parseString(datastring).childNodes[0] - return {node.nodeName: self._from_xml_node(node, plurals)} - except expat.ExpatError: - msg = _("cannot understand XML") - raise exception.MalformedRequestBody(reason=msg) - - def _from_xml_node(self, node, listnames): - """Convert a minidom node to a simple Python type. - - :param listnames: list of XML node names whose subnodes should - be considered list items. - - """ - - if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3: - return node.childNodes[0].nodeValue - elif node.nodeName in listnames: - return [self._from_xml_node(n, listnames) for n in node.childNodes] - else: - result = dict() - for attr in node.attributes.keys(): - result[attr] = node.attributes[attr].nodeValue - for child in node.childNodes: - if child.nodeType != node.TEXT_NODE: - result[child.nodeName] = self._from_xml_node(child, - listnames) - return result - - def find_first_child_named(self, parent, name): - """Search a nodes children for the first child with a given name""" - for node in parent.childNodes: - if node.nodeName == name: - return node - return None - - def find_children_named(self, parent, name): - """Return all of a nodes children who have the given name""" - for node in parent.childNodes: - if node.nodeName == name: - yield node - - def extract_text(self, node): - """Get the text field contained by the given node""" - if len(node.childNodes) == 1: - child = node.childNodes[0] - if child.nodeType == child.TEXT_NODE: - return child.nodeValue - return "" - - def default(self, datastring): - return {'body': self._from_xml(datastring)} diff --git a/bufunfa/plugin.py b/bufunfa/plugin.py deleted file mode 100644 index 74813e7..0000000 --- a/bufunfa/plugin.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import abc -from stevedore import driver - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common.loopingcall import LoopingCall -from bufunfa import exceptions - - -LOG = logging.getLogger(__name__) - - -class Plugin(object): - __metaclass__ = abc.ABCMeta - - __plugin_name__ = None - __plugin_type__ = None - - def __init__(self): - self.name = self.get_canonical_name() - LOG.debug("Loaded plugin %s", self.name) - self.tasks = [] - - def is_enabled(self): - """ - Is this Plugin enabled? - - :retval: Boolean - """ - return True - - @classmethod - def get_plugin(cls, name, ns=None, conf=None, invoke_on_load=False, - invoke_args=(), invoke_kwds={}): - """ - Load a plugin from namespace - """ - ns = ns or cls.__plugin_ns__ - if ns is None: - raise RuntimeError('No namespace provided or __plugin_ns__ unset') - LOG.debug('Looking for plugin %s in %s', name, ns) - mgr = driver.DriverManager(ns, name) - if conf: - mgr.driver.register_opts(conf) - return mgr.driver(*invoke_args, **invoke_kwds) if invoke_on_load \ - else mgr.driver - - @classmethod - def get_canonical_name(cls): - """ - Return the plugin name - """ - type_ = cls.get_plugin_type() - name = cls.get_plugin_name() - return "%s:%s" % (type_, name) - - @classmethod - def get_plugin_name(cls): - return cls.__plugin_name__ - - @classmethod - def get_plugin_type(cls): - return cls.__plugin_type__ - - @classmethod - def register_group_opts(cls, conf, group_name=None, opts=None): - """ - Register a set of Options underneath a new Group or Section - if you will. - - :param conf: Configuration object - :param group_name: Optional group name to register this under - Default: ClassName to class_name - :param opts: The options to register. - """ - group_name = group_name or cls.get_canonical_name() - if not group_name: - raise exceptions.ConfigurationError("Missing name") - - # NOTE(zykes): Always register the group if not the init fails... - group = cfg.OptGroup( - name=group_name, - title="Configuration for %s" % group_name) - conf.register_group(group) - if opts: - conf.register_opts(opts, group=group) - else: - LOG.debug("No options for %s, skipping registration", group_name) - - @classmethod - def register_opts(cls, conf): - """ - Register the options for this Plugin using the options from - cls.get_opts() as a default - - :param conf: Configration object - """ - opts = cls.get_opts() - cls.register_group_opts(conf, opts=opts) - - @classmethod - def get_opts(cls): - """ - Return a list of options for this plugin to be registered underneath - it's section - """ - return [] - - def start(self): - """ - Start this plugin - """ - - def stop(self): - """ - Stop this plugin from doing anything - """ - for task in self.tasks: - task.stop() - - def start_periodic(self, func, interval, initial_delay=None, - raise_on_error=False): - initial_delay = initial_delay or interval - - task = LoopingCall(func) - task.start(interval=interval, initial_delay=initial_delay) - return task diff --git a/bufunfa/recorder/__init__.py b/bufunfa/recorder/__init__.py deleted file mode 100644 index 726e1f5..0000000 --- a/bufunfa/recorder/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/bufunfa/recorder/base.py b/bufunfa/recorder/base.py deleted file mode 100644 index c89e861..0000000 --- a/bufunfa/recorder/base.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import abc - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common.context import get_admin_context -from bufunfa.plugin import Plugin - - -class RecordEngine(Plugin): - """ - Base Record engine for getting Records from external systems - """ - __plugin_type__ = 'recorder' - __plugin_ns__ = 'bufunfa.recorder' - - def __init__(self, record_service): - super(RecordEngine, self).__init__() - self.admin_context = get_admin_context() - self.record_service = record_service - - @classmethod - def get_opts(cls): - return [ - cfg.IntOpt('poll_interval', default=60), - cfg.BoolOpt('record_audit_logging', default=False, - help='Logs individual records pr get_records()')] - - @abc.abstractmethod - def process_records(self): - """ - Process this recorder engines billable records in the below format - and pass them to central for storage - { - "resource_id": "0cc13414-905d-4563-b61a-e80702566fd5", - "type": "instance", - "volume": 3.41, - "metadata": "{'test': 1}", - "start_timestamp": "2012-10-31T08:29:29.574000", - "end_timestamp": "2012-10-31T08:29:45.574000", - "system_account_id": "c97027dd880d4c129ae7a4ba7edade05" - } - - resource_id: The ID of the resource that's billed - (External ID typically) - type: The type, application, instance, network etc - volume: The volume that's currently pulled - metadata: JSON - start_timestamp: Start of the pulling period - end_timestamp: End of the pulling period - system_account_id: The account id in the external system - """ diff --git a/bufunfa/recorder/impl_ceilometer.py b/bufunfa/recorder/impl_ceilometer.py deleted file mode 100644 index 6e3ce3b..0000000 --- a/bufunfa/recorder/impl_ceilometer.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied verbatim from ceilometerclient -from datetime import datetime - -import ceilometerclient - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log -from bufunfa.openstack.common import timeutils -from bufunfa.openstack.common.rpc.common import RemoteError -from bufunfa.central import api as central_api -from bufunfa.recorder.openstack import OpenstackEngine - - -LOG = log.getLogger(__name__) - - -class RecordEngine(OpenstackEngine): - __plugin_name__ = 'ceilometer' - - def start(self): - self.periodic = self.start_periodic( - self.process_records, - cfg.CONF[self.name].poll_interval) - - def get_client(self): - """ - Get a ceilometerclient - """ - keystone_client = self.get_ksclient() - return ceilometerclient.Client(keystone_client=keystone_client) - - def process_records(self): - """ - Get the records between a period of time - """ - # NOTE (zykes): Needs cleaning - try: - self.client = self.get_client() - projects = self.client.get_projects() - except Exception, e: - LOG.exception(e) - return - - for project_id in projects: - if project_id is None: - continue - - started = datetime.now() - - start_timestamp = self.get_poll_start(project_id) - - project_records = self.get_project_records_between( - project_id, - start_timestamp=start_timestamp) - central_api.process_records(self.admin_context, project_records) - - central_api.set_polled_at(self.admin_context, project_id, started) - - self.record_service.publish_records(self.admin_context, - project_records) - - def get_poll_start(self, project_id): - """ - Get poll start time - - :param project_id: The project ID - """ - try: - account = central_api.get_system_account( - self.admin_context, - project_id) - except RemoteError: - return - # NOTE: The time sent over RPC is text. - polled_at = timeutils.parse_strtime(account['polled_at']) - return polled_at - - def get_project_records_between(self, project_id, start_timestamp=None, - end_timestamp=None): - """ - Get the given project id's records between given timestamps - - :param project_id: Project ID to get Records for. - :param start_timestamp: Start timestamp - :param end_timestamp: End timestamp - """ - - offset = cfg.CONF[self.name].poll_interval / 60 + 1 - resources = self.client.get_resources( - project_id, - start_timestamp=start_timestamp, - end_timestamp=end_timestamp, - search_offset=offset) - - records = [] - for resource in resources: - meters = [item.get('counter_name') for item in resource['meter']] - for meter in meters: - record = self.get_record_between( - resource, - meter, - start_timestamp=start_timestamp, - end_timestamp=end_timestamp) - if record is not None: - records.append(record) - LOG.debug("Returning %d records for project %s", len(records), - project_id) - return records - - def get_record_between(self, resource, meter, - start_timestamp=None, end_timestamp=None): - """ - Get a Record by resource and meter between x, y - - :param resource: A resource in Dict form - :param meter: Meter name - :param start_timestamp: Start timestamp - :param end_timestamp: End timestamp - """ - # NOTE: No type, skip it. Needs re-amp - type_, volume, metadata = self._get_meter_data(resource, meter) - if type_ is None: - return - - duration_info = self.client.get_resource_duration_info( - resource_id=resource['resource_id'], meter=meter, - start_timestamp=start_timestamp, end_timestamp=end_timestamp - ) - - #if not duration_info['start_timestamp'] and \ - # not duration_info['end_timestamp']: - #return - - volume = volume or duration_info.get('duration') - - # NOTE: Not sure on this but I think we can skip returning events that - # don't have volume or duration - if not volume and not duration_info.get('duration'): - return - - record = dict( - resource_id=resource['resource_id'], - account_id=resource['project_id'], - type=type_, - volume=volume, - extra=metadata, - start_timestamp=duration_info.get('start_timestamp'), - end_timestamp=duration_info.get('end_timestamp'), - duration=duration_info.get('duration') - ) - if cfg.CONF[self.name].record_audit_logging: - LOG.debug("Record: %s", record) - return record - - def _get_meter_data(self, resource, meter): - """ - :param resource: A resource in Dict form - :param meter: Meter name - """ - type_ = None - volume = resource['metadata'].get('size') - metadata = {} - - if meter.startswith('instance:'): - type_ = 'instance' - metadata['flavor'] = meter.partition(':')[-1] - elif meter == 'volume.size': - type_ = 'volume' - volume = self.client.get_resource_volume_max( - resource_id=resource['resource_id'], - meter=meter, - ) - elif meter == 'image.size': - type_ = 'image' - elif meter == 'network': - type_ = 'network' - elif meter == 'subnet': - type_ = 'subnet' - metadata['network_id'] = resource['metadata'].get('network_id') - metadata['cidr'] = resource['metadata'].get('cidr') - elif meter == 'port': - type_ = 'port' - metadata['network_id'] = resource['metadata'].get('network_id') - metadata['mac'] = resource['metadata'].get('mac_address') - ips = [] - for item in resource['metadata'].get('fixed_ips', []): - if 'ip_address' in item: - ips.append(item['ip_address']) - metadata['ips'] = ','.join(ips) - return type_, volume, metadata diff --git a/bufunfa/recorder/openstack.py b/bufunfa/recorder/openstack.py deleted file mode 100644 index 1cf0be2..0000000 --- a/bufunfa/recorder/openstack.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from urlparse import urlparse - -from keystoneclient.v2_0 import client as ksclient - -from bufunfa.openstack.common import cfg -from bufunfa.recorder.base import RecordEngine - - -class OpenstackEngine(RecordEngine): - def get_ksclient(self): - """ - Get a keystone client - """ - insecure = urlparse(cfg.CONF.os_auth_url).scheme != 'https' - return ksclient.Client(username=cfg.CONF.os_username, - password=cfg.CONF.os_password, - tenant_id=cfg.CONF.os_tenant_id, - tenant_name=cfg.CONF.os_tenant_name, - auth_url=cfg.CONF.os_auth_url, - insecure=insecure) diff --git a/bufunfa/recorder/service.py b/bufunfa/recorder/service.py deleted file mode 100644 index 92db6cd..0000000 --- a/bufunfa/recorder/service.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import os -from stevedore.named import NamedExtensionManager - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log -from bufunfa.openstack.common.context import get_admin_context -from bufunfa.openstack.common.rpc.service import Service -from bufunfa.central import api as central_api -from bufunfa.recorder.base import RecordEngine - - -LOG = log.getLogger(__name__) - -CLI_OPTIONS = [ - cfg.StrOpt('os-username', - default=os.environ.get('OS_USERNAME', 'glance'), - help='Username to use for openstack service access'), - cfg.StrOpt('os-password', - default=os.environ.get('OS_PASSWORD', 'admin'), - help='Password to use for openstack service access'), - cfg.StrOpt('os-tenant-id', - default=os.environ.get('OS_TENANT_ID', ''), - help='Tenant ID to use for openstack service access'), - cfg.StrOpt('os-tenant-name', - default=os.environ.get('OS_TENANT_NAME', 'admin'), - help='Tenant name to use for openstack service access'), - cfg.StrOpt('os-auth-url', - default=os.environ.get('OS_AUTH_URL', - 'http://localhost:5000/v2.0'), - help='Auth URL to use for openstack service access'), -] - -cfg.CONF.register_cli_opts(CLI_OPTIONS) - - -cfg.CONF.register_opts([ - cfg.ListOpt('record-engines', default=[], help="What engines to enable") -]) - - -class RecordService(Service): - def __init__(self, *args, **kw): - kw.update( - host=cfg.CONF.host, - topic=cfg.CONF.worker_topic) - - super(RecordService, self).__init__(*args, **kw) - - self.admin_context = get_admin_context() - - self.engines = self._init_extensions() - - def _init_extensions(self): - """ Loads and prepares all enabled extensions """ - self.extensions_manager = NamedExtensionManager( - RecordEngine.__plugin_ns__, names=cfg.CONF.record_engines) - - def _load_extension(ext): - handler_cls = ext.plugin - handler_cls.register_opts(cfg.CONF) - return handler_cls(record_service=self) - - try: - return self.extensions_manager.map(_load_extension) - except RuntimeError: - # No handlers enabled. No problem. - return [] - - def start(self): - """ - Start underlying engines - """ - super(RecordService, self).start() - for engine in self.engines: - engine.start() - - def stop(self): - """ - Stop underlying engines - """ - super(RecordService, self).stop() - for engine in self.engines: - engine.stop() - - def publish_records(self, context, records): - """ - Publish a record to the central service - - :param record: The record - """ - return central_api.process_records(context, records) diff --git a/bufunfa/service.py b/bufunfa/service.py deleted file mode 100644 index 4fe0eda..0000000 --- a/bufunfa/service.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- -# -# Copyright © 2012 eNovance -# -# Author: Julien Danjou -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import context -from bufunfa.openstack.common.rpc import service as rpc_service - - -""" -Module for base services -""" - - -cfg.CONF.register_opts([ - cfg.IntOpt('periodic_interval', - default=600, - help='seconds between running periodic tasks') -]) - - -class PeriodicService(rpc_service.Service): - """ - A Service that does stuff periodicly - """ - def start(self): - super(PeriodicService, self).start() - admin_context = context.RequestContext('admin', 'admin', is_admin=True) - self.tg.add_timer( - cfg.CONF.periodic_interval, - self.manager.periodic_tasks, - context=admin_context) diff --git a/bufunfa/storage/__init__.py b/bufunfa/storage/__init__.py deleted file mode 100644 index c2547cd..0000000 --- a/bufunfa/storage/__init__.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# Copyright © 2012 New Dream Network, LLC (DreamHost) -# -# Author: Doug Hellmann -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied from Ceilometer -from urlparse import urlparse -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log -from bufunfa.storage.base import StorageEngine - -LOG = log.getLogger(__name__) - -cfg.CONF.register_opts([ - cfg.StrOpt('database_connection', - default='sqlite:///$state_path/bufunfa.db', - help='The database driver to use'), -]) - - -def get_engine_name(string): - """ - Return the engine name from either a non-dialected or dialected string - """ - return string.split("+")[0] - - -def get_engine(conf): - scheme = urlparse(conf.database_connection).scheme - engine_name = get_engine_name(scheme) - return StorageEngine.get_plugin( - engine_name, conf=conf, invoke_on_load=True) - - -def get_connection(conf): - engine = get_engine(conf) - return engine.get_connection(conf) - - -def setup_schema(): - """ Create the DB - Used for testing purposes """ - connection = get_connection(cfg.CONF) - connection.setup_schema() - - -def teardown_schema(): - """ Reset the DB to default - Used for testing purposes """ - connection = get_connection(cfg.CONF) - connection.teardown_schema() diff --git a/bufunfa/storage/base.py b/bufunfa/storage/base.py deleted file mode 100644 index c03edcd..0000000 --- a/bufunfa/storage/base.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- encoding: utf-8 -*- -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import abc -from bufunfa.plugin import Plugin - - -class StorageEngine(Plugin): - """ - Base class for storage engines - """ - - __plugin_ns__ = 'bufunfa.storage' - __plugin_type__ = 'storage' - - @abc.abstractmethod - def get_connection(self, conf): - """ - Return a Connection instance based on the configuration settings. - """ - - -class Connection(object): - """ - A Connection - """ - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def __init__(self, conf): - """ - Constructor... - """ - - @abc.abstractmethod - def add_rate(self, context, values): - """ - Add a new rate - """ - - @abc.abstractmethod - def get_rates(self, context): - """ - Get all rates - """ - - @abc.abstractmethod - def update_rate(self, context, rate_id, values): - """ - Update a rate - """ - - @abc.abstractmethod - def delete_rate(self, context, rate_id): - """ - Delete a rate - """ diff --git a/bufunfa/storage/impl_sqlalchemy/__init__.py b/bufunfa/storage/impl_sqlalchemy/__init__.py deleted file mode 100644 index da2356a..0000000 --- a/bufunfa/storage/impl_sqlalchemy/__init__.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied from Moniker / Ceilometer -import copy - -from bufunfa import exceptions -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log -from bufunfa.storage import base -from bufunfa.storage.impl_sqlalchemy import models -from bufunfa.storage.impl_sqlalchemy.session import get_session - - -LOG = log.getLogger(__name__) - -SQL_OPTS = [ - cfg.IntOpt('connection_debug', default=50, - help='Verbosity of SQL debugging information. 0=None,' - ' 100=Everything'), - cfg.BoolOpt('connection_trace', default=False, - help='Add python stack traces to SQL as comment strings'), - cfg.BoolOpt('sqlite_synchronous', default=True, - help='If passed, use synchronous mode for sqlite'), - cfg.IntOpt('idle_timeout', default=3600, - help='timeout before idle sql connections are reaped'), - cfg.IntOpt('max_retries', default=10, - help='maximum db connection retries during startup. ' - '(setting -1 implies an infinite retry count)'), - cfg.IntOpt('retry_interval', default=10, - help='interval between retries of opening a sql connection') -] - - -class SQLAlchemyStorage(base.StorageEngine): - __plugin_name__ = 'sqlalchemy' - - @classmethod - def get_opts(cls): - opts = super(SQLAlchemyStorage, cls).get_opts() - opts.extend(SQL_OPTS) - return opts - - def get_connection(self, conf): - return Connection(conf) - - -class Connection(base.Connection): - """ - SQLAlchemy connection - """ - - def __init__(self, conf): - LOG.info('connecting to %s', conf.database_connection) - self.session = self._get_connection(conf) - # NOTE: Need to fix this properly... - self.setup_schema() - - def _get_connection(self, conf): - """ - Return a connection to the database. - """ - return get_session() - - def setup_schema(self): - """ Semi-Private Method to create the database schema """ - models.Base.metadata.create_all(self.session.bind) - - def teardown_schema(self): - """ Semi-Private Method to reset the database schema """ - models.Base.metadata.drop_all(self.session.bind) - - def _get_id(self, model, context, id_): - """ - Helper to not write the same code x times - """ - query = self.session.query(model) - obj = query.get(id_) - if not obj: - raise exceptions.NotFound(id_) - else: - return obj - - def _add(self, model, context, values): - obj = model() - obj.update(values) - obj.save(self.session) - return obj - - def _update(self, model, context, id, values): - obj = self._get_id(model, context, id) - obj.update(values) - try: - obj.save(self.session) - except exceptions.Duplicate: - raise - return dict(obj) - - def _add_or_update(self, context, model, values, id=None): - if id is None: - return self._add(context, model, values) - else: - return self._update(context, model, values, id) - - def add_rate(self, context, values): - return self._add(models.Rate, context, values) - - def get_rate(self, context, rate_id): - return self._get_id(models.Rate, context, rate_id) - - def get_rates(self, context): - query = self.session.query(models.Rate) - return [row2dict(row) for row in query.all()] - - def update_rate(self, context, rate_id, values): - return self._update(models.Rate, context, rate_id, values) - - def delete_rate(self, context, rate_id): - obj = self._get_id(models.Rate, context, rate_id) - obj.delete(self.session) - - def add_account(self, context, values): - return self._add(models.Account, context, values) - - def get_account(self, context, account_id): - return self._get_id(models.Account, context, account_id) - - def get_accounts(self, conetxt): - query = self.session.query(models.Account) - return [row2dict(row) for row in query.all()] - - def update_account(self, context, account_id, values): - return self._update(models.Account, context, account_id, values) - - def delete_account(self, context, account_id): - obj = self._get_id(models.Account, context, account_id) - obj.delete(self.session) - - def add_system_account(self, context, values): - return self._add(models.SystemAccount, context, values) - - def get_system_account(self, context, account_id): - return self._get_id(models.SystemAccount, context, account_id) - - def get_system_accounts(self, context): - query = self.session.query(models.SystemAccount) - return [row2dict(row) for row in query.all()] - - def update_system_account(self, context, account_id, values): - return self._update(models.SystemAccount, context, account_id, values) - - def delete_system_account(self, context, account_id): - obj = self._get_id(models.SystemAccount, context, account_id) - obj.delete(self.session) - - # NOTE: Records - def add_record(self, context, values): - self._add(models.Record, context, values) - - -def row2dict(row): - d = copy.copy(row.__dict__) - for col in ['_sa_instance_state']: - if col in d: - del d[col] - return d diff --git a/bufunfa/storage/impl_sqlalchemy/models.py b/bufunfa/storage/impl_sqlalchemy/models.py deleted file mode 100644 index 8eab08f..0000000 --- a/bufunfa/storage/impl_sqlalchemy/models.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied verbatim from Moniker -from uuid import uuid4 -from urlparse import urlparse - -from sqlalchemy import Column, DateTime, Unicode, Float, ForeignKey -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import relationship, object_mapper -from sqlalchemy.ext.declarative import declarative_base - -from bufunfa import exceptions -import bufunfa.openstack.common.cfg as cfg -from bufunfa.openstack.common import log -from bufunfa.openstack.common import timeutils -from bufunfa.storage.impl_sqlalchemy.types import JSONBlob, UUID - -LOG = log.getLogger(__name__) - -sql_opts = [ - cfg.IntOpt('mysql_engine', default='InnoDB', help='MySQL engine') -] - -cfg.CONF.register_opts(sql_opts) - - -def table_args(): - engine_name = urlparse(cfg.CONF.database_connection).scheme - if engine_name == 'mysql': - return {'mysql_engine': cfg.CONF.mysql_engine} - return None - - -class Base(object): - __abstract__ = True - - id = Column(UUID, default=uuid4, primary_key=True) - - created_at = Column(DateTime, default=timeutils.utcnow) - updated_at = Column(DateTime, onupdate=timeutils.utcnow) - - __table_args__ = table_args() - __table_initialized__ = False - - def save(self, session): - """ Save this object """ - session.add(self) - - try: - session.flush() - except IntegrityError, e: - if 'is not unique' in str(e): - raise exceptions.Duplicate(str(e)) - else: - raise - - def delete(self, session): - """ Delete this object """ - session.delete(self) - session.flush() - - def __setitem__(self, key, value): - setattr(self, key, value) - - def __getitem__(self, key): - return getattr(self, key) - - def __iter__(self): - columns = dict(object_mapper(self).columns).keys() - # NOTE(russellb): Allow models to specify other keys that can be looked - # up, beyond the actual db columns. An example would be the 'name' - # property for an Instance. - if hasattr(self, '_extra_keys'): - columns.extend(self._extra_keys()) - self._i = iter(columns) - return self - - def next(self): - n = self._i.next() - return n, getattr(self, n) - - def update(self, values): - """ Make the model object behave like a dict """ - for k, v in values.iteritems(): - setattr(self, k, v) - - def iteritems(self): - """ - Make the model object behave like a dict. - - Includes attributes from joins. - """ - local = dict(self) - joined = dict([(k, v) for k, v in self.__dict__.iteritems() - if not k[0] == '_']) - local.update(joined) - return local.iteritems() - - -Base = declarative_base(cls=Base) - - -class Account(Base): - """ - A way to correlate multiple tenants or future Domains in OpenStack into - a single aggregation point - """ - __tablename__ = 'accounts' - name = Column(Unicode(100), nullable=False) - - -class SystemAccount(Base): - """ - Bind a System's Account representation to a Account - - Examples: - OpenStack Domain or Tenant to a Account - Credit card Account representation to a Account - - id: The ID of the account in the System - name: The name of the system - - account_id: The ID of the Account internally - """ - __tablename__ = "system_accounts" - - id = Column(Unicode(40), primary_key=True) - name = Column(Unicode(100)) - polled_at = Column(DateTime) - - account = relationship("Account", backref="systems") - account_id = Column(UUID, ForeignKey('accounts.id')) - - -class Record(Base): - __tablename__ = 'records' - - resource_id = Column(Unicode(80), nullable=False) - type = Column(Unicode(80), nullable=False) - volume = Column(Float, nullable=False) - extra = Column(JSONBlob, nullable=True) - start_timestamp = Column(DateTime) - end_timestamp = Column(DateTime) - - account = relationship("SystemAccount", backref="records") - account_id = Column(Unicode(100), - ForeignKey('system_accounts.id'), - nullable=False) - - -class Rate(Base): - """ - The rate to charge for something - """ - __tablename__ = 'rates' - - name = Column(Unicode(60), nullable=False, unique=True) - value = Column(Float, nullable=False) diff --git a/bufunfa/storage/impl_sqlalchemy/session.py b/bufunfa/storage/impl_sqlalchemy/session.py deleted file mode 100644 index 146bd46..0000000 --- a/bufunfa/storage/impl_sqlalchemy/session.py +++ /dev/null @@ -1,225 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied from Ceilometer / Moniker -"""Session Handling for SQLAlchemy backend.""" - -import re -import time - -import sqlalchemy -from sqlalchemy.exc import DisconnectionError, OperationalError -import sqlalchemy.orm -from sqlalchemy.pool import NullPool, StaticPool - -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common.gettextutils import _ - -LOG = logging.getLogger(__name__) - -_MAKER = None -_ENGINE = None - -sql_opts = [ - cfg.IntOpt('sql_connection_debug', default=0, - help='Verbosity of SQL debugging information. 0=None,' - ' 100=Everything'), - cfg.BoolOpt('sql_connection_trace', default=False, - help='Add python stack traces to SQL as comment strings'), - cfg.BoolOpt('sqlite_synchronous', default=True, - help='If passed, use synchronous mode for sqlite'), - cfg.IntOpt('sql_idle_timeout', default=3600, - help='timeout before idle sql connections are reaped'), - cfg.IntOpt('sql_max_retries', default=10, - help='maximum db connection retries during startup. ' - '(setting -1 implies an infinite retry count)'), - cfg.IntOpt('sql_retry_interval', default=10, - help='interval between retries of opening a sql connection') -] - -cfg.CONF.register_opts(sql_opts) - - -def get_session(autocommit=True, expire_on_commit=False, autoflush=True): - """Return a SQLAlchemy session.""" - global _MAKER - - if _MAKER is None: - engine = get_engine() - _MAKER = get_maker(engine, autocommit, expire_on_commit, autoflush) - - session = _MAKER() - return session - - -def synchronous_switch_listener(dbapi_conn, connection_rec): - """Switch sqlite connections to non-synchronous mode""" - dbapi_conn.execute("PRAGMA synchronous = OFF") - - -def add_regexp_listener(dbapi_con, con_record): - """Add REGEXP function to sqlite connections.""" - - def regexp(expr, item): - reg = re.compile(expr) - return reg.search(unicode(item)) is not None - dbapi_con.create_function('regexp', 2, regexp) - - -def ping_listener(dbapi_conn, connection_rec, connection_proxy): - """ - Ensures that MySQL connections checked out of the - pool are alive. - - Borrowed from: - http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f - """ - try: - dbapi_conn.cursor().execute('select 1') - except dbapi_conn.OperationalError, ex: - if ex.args[0] in (2006, 2013, 2014, 2045, 2055): - LOG.warn('Got mysql server has gone away: %s', ex) - raise DisconnectionError("Database server went away") - else: - raise - - -def is_db_connection_error(args): - """Return True if error in connecting to db.""" - # NOTE(adam_g): This is currently MySQL specific and needs to be extended - # to support Postgres and others. - conn_err_codes = ('2002', '2003', '2006') - for err_code in conn_err_codes: - if args.find(err_code) != -1: - return True - return False - - -def get_engine(): - """Return a SQLAlchemy engine.""" - global _ENGINE - if _ENGINE is None: - connection_dict = sqlalchemy.engine.url.make_url( - cfg.CONF.database_connection) - - engine_args = { - "pool_recycle": cfg.CONF.sql_idle_timeout, - "echo": False, - 'convert_unicode': True, - } - - # Map our SQL debug level to SQLAlchemy's options - if cfg.CONF.sql_connection_debug >= 100: - engine_args['echo'] = 'debug' - elif cfg.CONF.sql_connection_debug >= 50: - engine_args['echo'] = True - - if "sqlite" in connection_dict.drivername: - engine_args["poolclass"] = NullPool - - if cfg.CONF.database_connection == "sqlite://": - engine_args["poolclass"] = StaticPool - engine_args["connect_args"] = {'check_same_thread': False} - - _ENGINE = sqlalchemy.create_engine(cfg.CONF.database_connection, - **engine_args) - - if 'mysql' in connection_dict.drivername: - sqlalchemy.event.listen(_ENGINE, 'checkout', ping_listener) - elif "sqlite" in connection_dict.drivername: - if not cfg.CONF.sqlite_synchronous: - sqlalchemy.event.listen(_ENGINE, 'connect', - synchronous_switch_listener) - sqlalchemy.event.listen(_ENGINE, 'connect', add_regexp_listener) - - if (cfg.CONF.sql_connection_trace and - _ENGINE.dialect.dbapi.__name__ == 'MySQLdb'): - import MySQLdb.cursors - _do_query = debug_mysql_do_query() - setattr(MySQLdb.cursors.BaseCursor, '_do_query', _do_query) - - try: - _ENGINE.connect() - except OperationalError, e: - if not is_db_connection_error(e.args[0]): - raise - - remaining = cfg.CONF.sql_max_retries - if remaining == -1: - remaining = 'infinite' - while True: - msg = _('SQL connection failed. %s attempts left.') - LOG.warn(msg % remaining) - if remaining != 'infinite': - remaining -= 1 - time.sleep(cfg.CONF.sql_retry_interval) - try: - _ENGINE.connect() - break - except OperationalError, e: - if (remaining != 'infinite' and remaining == 0) or \ - not is_db_connection_error(e.args[0]): - raise - return _ENGINE - - -def get_maker(engine, autocommit=True, expire_on_commit=False, autoflush=True): - """Return a SQLAlchemy sessionmaker using the given engine.""" - return sqlalchemy.orm.sessionmaker(bind=engine, - autocommit=autocommit, - autoflush=autoflush, - expire_on_commit=expire_on_commit) - - -def debug_mysql_do_query(): - """Return a debug version of MySQLdb.cursors._do_query""" - import MySQLdb.cursors - import traceback - - old_mysql_do_query = MySQLdb.cursors.BaseCursor._do_query - - def _do_query(self, q): - stack = '' - for file, line, method, function in traceback.extract_stack(): - # exclude various common things from trace - if file.endswith('session.py') and method == '_do_query': - continue - if file.endswith('api.py') and method == 'wrapper': - continue - if file.endswith('utils.py') and method == '_inner': - continue - if file.endswith('exception.py') and method == '_wrap': - continue - # nova/db/api is just a wrapper around nova/db/sqlalchemy/api - if file.endswith('nova/db/api.py'): - continue - # only trace inside nova - index = file.rfind('nova') - if index == -1: - continue - stack += "File:%s:%s Method:%s() Line:%s | " \ - % (file[index:], line, method, function) - - # strip trailing " | " from stack - if stack: - stack = stack[:-3] - qq = "%s /* %s */" % (q, stack) - else: - qq = q - old_mysql_do_query(self, qq) - - # return the new _do_query method - return _do_query diff --git a/bufunfa/storage/impl_sqlalchemy/types.py b/bufunfa/storage/impl_sqlalchemy/types.py deleted file mode 100644 index 10109df..0000000 --- a/bufunfa/storage/impl_sqlalchemy/types.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2012 Bouvet ASA -# Copyright 2012 Managed I.T. -# -# Author: Endre Karlson -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied UUID from Moniker -from sqlalchemy.types import TypeDecorator, CHAR, Text -from sqlalchemy.dialects.postgresql import UUID as pgUUID -from bufunfa.openstack.common import jsonutils as json -import uuid - - -class UUID(TypeDecorator): - """Platform-independent UUID type. - - Uses Postgresql's UUID type, otherwise uses - CHAR(32), storing as stringified hex values. - - Copied verbatim from SQLAlchemy documentation. - """ - impl = CHAR - - def load_dialect_impl(self, dialect): - if dialect.name == 'postgresql': - return dialect.type_descriptor(pgUUID()) - else: - return dialect.type_descriptor(CHAR(32)) - - def process_bind_param(self, value, dialect): - if value is None: - return value - elif dialect.name == 'postgresql': - return str(value) - else: - if not isinstance(value, uuid.UUID): - return "%.32x" % uuid.UUID(value) - else: - # hexstring - return "%.32x" % value - - def process_result_value(self, value, dialect): - if value is None: - return value - else: - return uuid.UUID(value) - - -class JSONBlob(TypeDecorator): - """ - A way to implement JSON - """ - impl = Text - - def process_bind_param(self, value, dialect): - return json.dumps(value) - - def process_result_value(self, value, dialect): - return json.loads(value) diff --git a/bufunfa/storage/impl_sqlalchemy/utils.py b/bufunfa/storage/impl_sqlalchemy/utils.py deleted file mode 100644 index 341b015..0000000 --- a/bufunfa/storage/impl_sqlalchemy/utils.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2010-2011 OpenStack LLC. -# Copyright 2012 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# NOTE(kiall): Copied verbatim from Nova - -"""Implementation of paginate query.""" - -import sqlalchemy - -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common.gettextutils import _ -from bufunfa import exceptions - - -LOG = logging.getLogger(__name__) - - -# copy from glance/db/sqlalchemy/api.py -def paginate_query(query, model, limit, sort_keys, marker=None, - sort_dir=None, sort_dirs=None): - """Returns a query with sorting / pagination criteria added. - - Pagination works by requiring a unique sort_key, specified by sort_keys. - (If sort_keys is not unique, then we risk looping through values.) - We use the last row in the previous page as the 'marker' for pagination. - So we must return values that follow the passed marker in the order. - With a single-valued sort_key, this would be easy: sort_key > X. - With a compound-values sort_key, (k1, k2, k3) we must do this to repeat - the lexicographical ordering: - (k1 > X1) or (k1 == X1 && k2 > X2) or (k1 == X1 && k2 == X2 && k3 > X3) - - We also have to cope with different sort_directions. - - Typically, the id of the last row is used as the client-facing pagination - marker, then the actual marker object must be fetched from the db and - passed in to us as marker. - - :param query: the query object to which we should add paging/sorting - :param model: the ORM model class - :param limit: maximum number of items to return - :param sort_keys: array of attributes by which results should be sorted - :param marker: the last item of the previous page; we returns the next - results after this value. - :param sort_dir: direction in which results should be sorted (asc, desc) - :param sort_dirs: per-column array of sort_dirs, corresponding to sort_keys - - :rtype: sqlalchemy.orm.query.Query - :return: The query with sorting/pagination added. - """ - - if 'id' not in sort_keys: - # TODO(justinsb): If this ever gives a false-positive, check - # the actual primary key, rather than assuming its id - LOG.warn(_('Id not in sort_keys; is sort_keys unique?')) - - assert(not (sort_dir and sort_dirs)) - - # Default the sort direction to ascending - if sort_dirs is None and sort_dir is None: - sort_dir = 'asc' - - # Ensure a per-column sort direction - if sort_dirs is None: - sort_dirs = [sort_dir for _sort_key in sort_keys] - - assert(len(sort_dirs) == len(sort_keys)) - - # Add sorting - for current_sort_key, current_sort_dir in zip(sort_keys, sort_dirs): - sort_dir_func = { - 'asc': sqlalchemy.asc, - 'desc': sqlalchemy.desc, - }[current_sort_dir] - - try: - sort_key_attr = getattr(model, current_sort_key) - except AttributeError: - raise exceptions.InvalidSortKey() - query = query.order_by(sort_dir_func(sort_key_attr)) - - # Add pagination - if marker is not None: - marker_values = [] - for sort_key in sort_keys: - v = getattr(marker, sort_key) - marker_values.append(v) - - # Build up an array of sort criteria as in the docstring - criteria_list = [] - for i in xrange(0, len(sort_keys)): - crit_attrs = [] - for j in xrange(0, i): - model_attr = getattr(model, sort_keys[j]) - crit_attrs.append((model_attr == marker_values[j])) - - model_attr = getattr(model, sort_keys[i]) - if sort_dirs[i] == 'desc': - crit_attrs.append((model_attr < marker_values[i])) - elif sort_dirs[i] == 'asc': - crit_attrs.append((model_attr > marker_values[i])) - else: - raise ValueError(_("Unknown sort direction, " - "must be 'desc' or 'asc'")) - - criteria = sqlalchemy.sql.and_(*crit_attrs) - criteria_list.append(criteria) - - f = sqlalchemy.sql.or_(*criteria_list) - query = query.filter(f) - - if limit is not None: - query = query.limit(limit) - - return query diff --git a/bufunfa/tests/__init__.py b/bufunfa/tests/__init__.py deleted file mode 100644 index def9cc1..0000000 --- a/bufunfa/tests/__init__.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# NOTE(zykes): Copied from Moniker -import sys -import unittest -import mox -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common import context -from bufunfa import storage - -LOG = logging.getLogger(__name__) - - -class TestCase(unittest.TestCase): - def setUp(self): - super(TestCase, self).setUp() - - self.mox = mox.Mox() - self.config(database_connection='sqlite://', - rpc_backend='bufunfa.openstack.common.rpc.impl_fake', - notification_driver=[]) - storage.setup_schema() - - def tearDown(self): - storage.teardown_schema() - cfg.CONF.reset() - self.mox.UnsetStubs() - - super(TestCase, self).tearDown() - - def config(self, **kwargs): - group = kwargs.pop('group', None) - for k, v in kwargs.iteritems(): - cfg.CONF.set_override(k, v, group) - - def get_context(self, **kwargs): - return context.RequestContext(**kwargs) - - def get_admin_context(self): - return context.get_admin_context() - - -if sys.version_info < (2, 7): - # Add in some of the nicer methods not present in 2.6 - from contextlib import contextmanager - - def assertIsNone(self, expr, msg=None): - return self.assertEqual(expr, None, msg) - - TestCase.assertIsNone = assertIsNone - - def assertIsNotNone(self, expr, msg=None): - return self.assertNotEqual(expr, None, msg) - - TestCase.assertIsNotNone = assertIsNotNone - - def assertRaises(self, excClass, callableObj=None, *args, **kwargs): - @contextmanager - def context(): - raised = None - try: - yield - except Exception, e: - raised = e - finally: - if not isinstance(raised, excClass): - raise self.failureException( - "%s not raised" % str(excClass)) - - if callableObj is None: - return context() - with context: - callableObj(*args, **kwargs) - - TestCase.assertRaises = assertRaises diff --git a/bufunfa/tests/test_central/__init__.py b/bufunfa/tests/test_central/__init__.py deleted file mode 100644 index 535f930..0000000 --- a/bufunfa/tests/test_central/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from bufunfa.central import service as central_service -from bufunfa.tests import TestCase -from bufunfa.api import service as api_service - - -class CentralTestCase(TestCase): - __test__ = False - - def get_central_service(self): - return central_service.Service() - - def get_api_service(self): - return api_service.Service() diff --git a/bufunfa/tests/test_central/test_service.py b/bufunfa/tests/test_central/test_service.py deleted file mode 100644 index e35e225..0000000 --- a/bufunfa/tests/test_central/test_service.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from datetime import datetime, timedelta -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common import timeutils -from bufunfa.tests.test_central import CentralTestCase -from bufunfa import exceptions - -LOG = logging.getLogger(__name__) - - -class ServiceTest(CentralTestCase): - __test__ = True - - record = { - 'resource_id': '0cc13414-905d-4563-b61a-e80702566fd5', - 'type': 'instance', - 'volume': 3.5, - 'start_timestamp': datetime.now() - timedelta(1), - 'end_timestamp': datetime.now(), - 'account_id': 'c97027dd880d4c129ae7a4ba7edade05' - } - - rates = [ - {'name': 'cpu', 'value': 1}, - {'name': 'memory', 'value': 2} - ] - - accounts = [ - {'name': 'customer_a'} - ] - - system_accounts = [ - {'name': 'system_a', 'id': 'd44f1779-5034-455e-b334-cac2ac3eee33'}, - {'name': 'system_b', 'id': 'a45e43af-090b-4045-ae78-6a9d507d1418'} - ] - - def setUp(self): - super(ServiceTest, self).setUp() - self.service = self.get_central_service() - self.admin_context = self.get_admin_context() - - def add_rate(self, fixture=0, context=None, values={}): - context = context or self.get_admin_context() - values = self.rates[fixture] - values.update(values) - return self.service.add_rate(context, values) - - def add_account(self, fixture=0, context=None, values={}): - context = context or self.get_admin_context() - values = self.accounts[fixture] - values.update(values) - return self.service.add_account(context, values) - - def add_system_account(self, fixture=0, context=None, values={}): - context = context or self.get_admin_context() - values = self.system_accounts[fixture] - values.update(values) - return self.service.add_system_account(context, values) - - def test_process_record_unexisting_system(self): - """ - If the system we we're receiving a record from doesn't have a system - account entry we'll create one - """ - self.service.process_record( - self.admin_context, self.record) - - system = self.service.storage_conn.get_system_account( - self.admin_context, self.record['account_id']) - self.assertEquals(system.id, self.record['account_id']) - - def test_set_polled_at(self): - """ - Set the last time the SystemAccount was polled - """ - account_id = str(self.add_system_account()['id']) - now = datetime.now() - self.service.set_polled_at(self.admin_context, account_id, - timeutils.strtime(now)) - - account = self.service.get_system_account(self.admin_context, - account_id) - self.assertEquals(account["polled_at"], now) - - def test_set_polled_at_too_old(self): - """ - Shouldn't be allowed to set polled_at older then the current one in - SystemAccount - """ - account_id = str(self.add_system_account()['id']) - now = datetime.now() - self.service.set_polled_at( - self.admin_context, account_id, timeutils.strtime(now)) - - with self.assertRaises(exceptions.TooOld): - self.service.set_polled_at( - self.admin_context, account_id, - timeutils.strtime(now - timedelta(1))) diff --git a/bufunfa/tests/test_storage/__init__.py b/bufunfa/tests/test_storage/__init__.py deleted file mode 100644 index ee7d1b9..0000000 --- a/bufunfa/tests/test_storage/__init__.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the 'License'); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import copy -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common import log as logging -from bufunfa.tests import TestCase -from bufunfa import storage -from bufunfa import exceptions - -LOG = logging.getLogger(__name__) - - -class StorageTestCase(TestCase): - __test__ = False - - def get_storage_driver(self, conf=cfg.CONF): - connection = storage.get_connection(conf) - return connection - - -class StorageDriverTestCase(StorageTestCase): - __test__ = False - - rate_fixtures = [ - {'name': 'cpu', 'value': 1}, - {'name': 'memory', 'value': 2} - ] - - account_fixtures = [ - {'name': 'customer_a'}, - {'name': 'customer_b'} - ] - - system_account_fixtures = [ - { - 'id': 'd44f1779-5034-455e-b334-cac2ac3eee33', - 'name': 'system_a' - }, - { - 'id': 'a45e43af-090b-4045-ae78-6a9d507d1418', - 'name': 'system_b' - } - ] - - def setUp(self): - super(StorageDriverTestCase, self).setUp() - self.storage_conn = self.get_storage_driver() - self.admin_context = self.get_admin_context() - - def test_init(self): - self.get_storage_driver() - - def add_rate_fixture(self, context=None, fixture=0, values={}): - context = context or self.admin_context - _values = copy.copy(self.rate_fixtures[fixture]) - _values.update(values) - return self.storage_conn.add_rate(context, _values) - - def add_account_fixture(self, context=None, fixture=0, values={}): - context = context or self.admin_context - _values = copy.copy(self.account_fixtures[fixture]) - _values.update(values) - return self.storage_conn.add_account(context, _values) - - def add_system_account_fixture(self, context=None, fixture=0, values={}): - context = context or self.admin_context - _values = copy.copy(self.system_account_fixtures[fixture]) - _values.update(values) - return self.storage_conn.add_system_account(context, _values) - - def test_add_rate(self): - rate = self.add_rate_fixture() - self.assertEquals(rate.name, self.rate_fixtures[0]['name']) - self.assertEquals(rate.value, self.rate_fixtures[0]['value']) - - def test_delete_rate(self): - rate = self.add_rate_fixture() - self.storage_conn.delete_rate(self.admin_context, rate.id) - with self.assertRaises(exceptions.NotFound): - self.storage_conn.get_rate(self.admin_context, rate.id) - - def test_update_rate(self): - rate = self.add_rate_fixture() - self.storage_conn.update_rate( - self.admin_context, - rate.id, - values={'name': 'memory', 'value': 15}) - self.assertEquals(rate.name, 'memory') - self.assertEquals(rate.value, 15) - - def test_add_account(self): - account = self.add_account_fixture() - self.assertEquals(account.name, self.account_fixtures[0]['name']) - - def test_delete_account(self): - account = self.add_account_fixture() - self.storage_conn.delete_account(self.admin_context, account.id) - with self.assertRaises(exceptions.NotFound): - self.storage_conn.get_account(self.admin_context, account.id) - - def test_update_account(self): - account = self.add_account_fixture() - self.storage_conn.update_account( - self.admin_context, - account.id, - values={'name': 'customer_a'}) - self.assertEquals(account.name, 'customer_a') - - def test_add_system_account(self): - account = self.add_system_account_fixture() - self.assertEquals(account.name, - self.system_account_fixtures[0]['name']) - - def test_delete_system_account(self): - account = self.add_system_account_fixture() - self.storage_conn.delete_system_account(self.admin_context, account.id) - with self.assertRaises(exceptions.NotFound): - self.storage_conn.get_system_account(self.admin_context, - account.id) - - def test_update_system_account(self): - account = self.add_system_account_fixture() - self.storage_conn.update_system_account( - self.admin_context, - account.id, - values={'name': 'system_b'}) - self.assertEquals(account.name, 'system_b') diff --git a/bufunfa/tests/test_storage/test_scheme.py b/bufunfa/tests/test_storage/test_scheme.py deleted file mode 100644 index 1dfec75..0000000 --- a/bufunfa/tests/test_storage/test_scheme.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from bufunfa.tests import TestCase -from bufunfa.storage import get_engine_name - - -class TestEngineName(TestCase): - def test_engine_non_dialected(self): - name = get_engine_name("mysql") - self.assertEqual(name, "mysql") - - def test_engine_dialacted(self): - name = get_engine_name("mysql+oursql") - self.assertEqual(name, "mysql") diff --git a/bufunfa/tests/test_storage/test_sqlalchemy.py b/bufunfa/tests/test_storage/test_sqlalchemy.py deleted file mode 100644 index 95d557a..0000000 --- a/bufunfa/tests/test_storage/test_sqlalchemy.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from bufunfa.openstack.common import log as logging -from bufunfa.tests.test_storage import StorageDriverTestCase - -LOG = logging.getLogger(__name__) - - -class SqlalchemyTest(StorageDriverTestCase): - __test__ = True - - def setUp(self): - super(SqlalchemyTest, self).setUp() - self.config(database_connection='sqlite://') diff --git a/bufunfa/utils.py b/bufunfa/utils.py deleted file mode 100644 index e54eccf..0000000 --- a/bufunfa/utils.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import os -from bufunfa.openstack.common import log as logging -from bufunfa.openstack.common import cfg -from bufunfa.openstack.common.notifier import api as notifier_api -from bufunfa import exceptions - -LOG = logging.getLogger(__name__) - - -def notify(context, service, event_type, payload): - priority = 'INFO' - publisher_id = notifier_api.publisher_id(service) - - notifier_api.notify(context, publisher_id, event_type, priority, payload) - - -def find_config(config_path): - """ - Find a configuration file using the given hint. - - Code nabbed from cinder. - - :param config_path: Full or relative path to the config. - :returns: Full path of the config, if it exists. - :raises: `bufunfa.exceptions.ConfigNotFound` - """ - possible_locations = [ - config_path, - os.path.join("etc", "bufunfa", config_path), - os.path.join("etc", config_path), - os.path.join(cfg.CONF.state_path, "etc", "bufunfa", config_path), - os.path.join(cfg.CONF.state_path, "etc", config_path), - os.path.join(cfg.CONF.state_path, config_path), - "/etc/bufunfa/%s" % config_path, - ] - - for path in possible_locations: - LOG.debug('Checking path: %s' % path) - if os.path.exists(path): - return os.path.abspath(path) - raise exceptions.ConfigNotFound(config_path) diff --git a/bufunfa/version.py b/bufunfa/version.py deleted file mode 100644 index 2f79a04..0000000 --- a/bufunfa/version.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from bufunfa.openstack.common import version as common_version - -PRE_VERSION = '2013.1' -version_info = common_version.VersionInfo('bufunfa', pre_version=PRE_VERSION) diff --git a/bufunfa/wsgi.py b/bufunfa/wsgi.py deleted file mode 100644 index 7029b8f..0000000 --- a/bufunfa/wsgi.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2012 Managed I.T. -# -# Author: Kiall Mac Innes -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from bufunfa.openstack.common import wsgi - - -class Middleware(wsgi.Middleware): - @classmethod - def factory(cls, global_config, **local_conf): - """ Used for paste app factories in paste.deploy config files """ - - def _factory(app): - return cls(app, **local_conf) - - return _factory diff --git a/doc/requirements.txt b/doc/requirements.txt deleted file mode 100644 index 713bc4a..0000000 --- a/doc/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -Flask==0.9 -iso8601>=0.1.4 -eventlet -sqlalchemy>=0.7 -jsonschema>=0.6 -PasteDeploy - -# Needed for Keystone Middleware -https://launchpad.net/keystone/folsom/2012.2/+download/keystone-2012.2.tar.gz#egg=keystone -python-keystoneclient - -# Optional Stuff that is used by default -kombu -nose -mox -coverage -pep8==1.3.3 -openstack.nose_plugin -sphinx -sphinxcontrib-httpdomain diff --git a/doc/source/architecture.rst b/doc/source/architecture.rst deleted file mode 100644 index d98971c..0000000 --- a/doc/source/architecture.rst +++ /dev/null @@ -1,51 +0,0 @@ -.. - Copyright 2012 Endre Karlson for Bouvet ASA - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _architecture: - - -============ -Architecture -============ - -.. index:: - double: architecture; brief - -Brief overview -++++++++++++++ - :term:`central` is the central component that stores data persistantly in - :term:`storage` which is a backend using typically SQLAlchemy, MongoDB or - others. - - :term:`recorder` is the component that will fetch records from external - datasources. This can look a bit like Ceilometer's agents but it's not - quite. It loads difference "plugins" which it uses to poll data and - transforms the data into a :term:`record`. - - :term:`api` standard OpenStack alike REST api service that communicates via - :term:`mq` to :term:`central` in order to do actions like adding - records and so on. - -Service layout -++++++++++++++ -Below you can see a picture of how Bufunfa's services work. - -.. image:: services.png - -Storage layout -++++++++++++++ -Below you can see a picture of how Bufunfa's storage is layed out. - -.. image:: storage_layout.png diff --git a/doc/source/conf.py b/doc/source/conf.py deleted file mode 100644 index 5969472..0000000 --- a/doc/source/conf.py +++ /dev/null @@ -1,243 +0,0 @@ -# -*- coding: utf-8 -*- -# -# bufunfa documentation build configuration file, created by -# sphinx-quickstart on Wed Oct 31 18:58:17 2012. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinxcontrib.httpdomain'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'bufunfa' -copyright = u'2012, Bouvet ASA' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -from bufunfa.version import version_info as bufunfa_version -version = bufunfa_version.canonical_version_string() -# The full version, including alpha/beta/rc tags. -release = bufunfa_version.version_string_with_vcs() - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'bufunfadoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'bufunfa.tex', u'bufunfa Documentation', - u'Bouvet ASA', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'bufunfa', u'bufunfa Documentation', - [u'Bouvet ASA'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'bufunfa', u'bufunfa Documentation', - u'Bouvet ASA', 'bufunfa', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst deleted file mode 100644 index 3d540c8..0000000 --- a/doc/source/configuration.rst +++ /dev/null @@ -1,131 +0,0 @@ -.. - Copyright 2012 Endre Karlson for Bouvet ASA - Copyright 2012 New Dream Network, LLC (DreamHost) - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _configuration: - -======================= - Configuration Options -======================= - -Bufunfa specific -================ - -=============================== ==================================== ============================================================== -Parameter Default Note -=============================== ==================================== ============================================================== -api_host 0.0.0.0 API listen host -api_port 0.0.0.0 API listen port -control_exchange bufunfa The MQ Control exchange -worker_topic worker The topic that the Recorder(s) should use -central_topic central The topic that the Central should use -os-username glance Username to use for openstack service access -os-password admin Password to use for openstack service access -os-tenant-id Tenant ID to use for openstack service access -os-tenant-name admin Tenant name to use for openstack service access -os-auth-url http://localhost:5000/v2.0 Auth URL to use for openstack service access -database_connection sqlite:///$pystatepath/bufunfa.db Database connection string -=============================== ==================================== ============================================================== - -SQL Alchemy -=========== - -========================== ==================================== ============================================================== -Parameter Default Note -========================== ==================================== ============================================================== -sql_connection_debug 0 Verbosity of SQL debugging information. 0=None, 100=Everything -sql_connection_trace False Add python stack traces to SQL as comment strings -sql_idle_timeout 3600 timeout before idle sql connections are reaped -sql_max_retries 10 maximum db connection retries during startup. - (setting -1 implies an infinite retry count) -sql_retry_interval 10 interval between retries of opening a sql connection -mysql_engine InnoDB MySQL engine to use -sqlite_synchronous True If passed, use synchronous mode for sqlite -========================== ==================================== ============================================================== - -General options -=============== - -The following is the list of openstack-common options that we use: - -=========================== ==================================== ============================================================== -Parameter Default Note -=========================== ==================================== ============================================================== -default_notification_level INFO Default notification level for outgoing notifications -default_publisher_id $host Default publisher_id for outgoing notifications -bind_host 0.0.0.0 IP address to listen on -bind_port 9001 Port numver to listen on -port 5672 Rabbit MQ port to liste on -fake_rabbit False If passed, use a fake RabbitMQ provider -publish_errors False publish error events -use_stderr True Log output to standard error -logfile_mode 0644 Default file mode used when creating log files -logdir Log output to a per-service log file in named directory -logfile Log output to a named file -default_log_levels ['amqplib=WARN',sqlalchemy=WARN,...] Default log level per components -notification_topics ['notifications', ] AMQP topic used for openstack notifications -enabled_apis ['ec2', 'osapi_compute'] List of APIs to enable by default -verbose False Print more verbose output -debug False Print debugging output -state_path currentdir Top-level directory for maintaining nova state -sqlite_db nova.sqlite file name for sqlite -matchmaker_ringfile /etc/bufunfa/matchmaker_ring.json Matchmaker ring file (JSON) -rpc_zmq_bind_address '*' ZeroMQ bind address -rpc_zmq_matchmaker bufunfa.openstack.common.rpc. MatchMaker drivers - matchmaker.MatchMakerLocalhost -rpc_zmq_port 9501 ZeroMQ receiver listening port -rpc_zmq_port_pub 9502 ZeroMQ fanout publisher port -rpc_zmq_contexts 1 Number of ZeroMQ contexts -rpc_zmq_ipc_dir /var/run/openstack Directory for holding IPC sockets -rabbit_port 5672 The RabbitMQ broker port where a single node is used -rabbit_host localhost The RabbitMQ broker address where a single node is used -rabbit_hosts ['$rabbit_host:$rabbit_port'] The list of rabbit hosts to listen to -rabbit_userid guest the RabbitMQ userid -rabbit_password guest the RabbitMQ password -rabbit_virtual_host / the RabbitMQ virtual host -rabbit_retry_interval 1 how frequently to retry connecting with RabbitMQ -rabbit_retry_backoff 2 how long to backoff for between retries when connecting -rabbit_max_retries 0 maximum retries with trying to connect to RabbitMQ - (the default of 0 implies an infinite retry count) -rabbit_durable_queues False use durable queues in RabbitMQ -rabbit_use_ssl False connect over SSL for RabbitMQ -rabbit_durable_queues False use durable queues in RabbitMQ -rabbit_ha_queues False use H/A queues in RabbitMQ (x-ha-policy: all). -kombu_ssl_version SSL version to use (valid only if SSL enabled) -kombu_ssl_keyfile SSL key file (valid only if SSL enabled) -kombu_ssl_certfile SSL cert file (valid only if SSL enabled) -kombu_ssl_ca_certs SSL certification authority file -qpid_hostname localhost Qpid broker hostname -qpid_port 5672 Qpid broker port -qpid_username Username for qpid connection -qpid_password Password for qpid connection -qpid_sasl_mechanisms Space separated list of SASL mechanisms to use for auth -qpid_reconnect_timeout 0 Reconnection timeout in seconds -qpid_reconnect_limit 0 Max reconnections before giving up -qpid_reconnect_interval_min 0 Minimum seconds between reconnection attempts -qpid_reconnect_interval_max 0 Maximum seconds between reconnection attempts -qpid_reconnect_interval 0 Equivalent to setting max and min to the same value -qpid_heartbeat 60 Seconds between connection keepalive heartbeats -qpid_protocol tcp Transport to use, either 'tcp' or 'ssl' -qpid_reconnect True Automatically reconnect -qpid_tcp_nodelay True Disable Nagle algorithm -rpc_backend kombu The messaging module to use, defaults to kombu. -rpc_thread_pool_size 64 Size of RPC thread pool -rpc_conn_pool_size 30 Size of RPC connection pool -rpc_response_timeout 60 Seconds to wait for a response from call or multicall -rpc_cast_timeout 30 Seconds to wait before a cast expires (TTL). - Only supported by impl_zmq. -=========================== ==================================== ============================================================== - diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst deleted file mode 100644 index 2d242c0..0000000 --- a/doc/source/glossary.rst +++ /dev/null @@ -1,42 +0,0 @@ -.. - Copyright 2012 Endre Karlson for Bouvet ASA - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _architecture: - - -============ -Glossary -============ - -.. glossary:: - recorder - Agent that polls or receives data, generalizes it according to - :term:`record` and sends it to :term:`central` for storage. - central - Software service running on a central management node that stores - information persistantly in a backend storage using a configurable driver - like SQLAlchemy or other. - api - HTTP REST API service for Bufunfa - mq - A message queue, typically something like RabbitMQ or ZeroMQ - storage - A backend for storing data/information persistantly. Typically MongoDB or - a SQL based server software. - - record - A Record is typically something that's been metered, the record is a - simple piece of information that holds general info that's needed to - process billing info + extra if wanted. diff --git a/doc/source/index.rst b/doc/source/index.rst deleted file mode 100644 index 45b32b3..0000000 --- a/doc/source/index.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. bufunfa documentation master file, created by - sphinx-quickstart on Wed Oct 31 18:58:17 2012. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to Bufunfa's documentation! -=================================== - -Contents: - -.. toctree:: - :maxdepth: 2 - - architecture - configuration - install - glossary - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/doc/source/install.rst b/doc/source/install.rst deleted file mode 100644 index 969cd65..0000000 --- a/doc/source/install.rst +++ /dev/null @@ -1,145 +0,0 @@ -.. - Copyright 2012 Endre Karlson for Bouvet ASA - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _install: - -======================== -Install -======================== - -bufunfa is comprised of three components for more info on these please -consolidate :architecture:. - -.. note:: - bufunfa makes extensive use of the messaging bus, but has not - yet been tested with ZeroMQ. We recommend using Rabbit or qpid - for now. - - -From Packages -+++++++++++++ - - -From Source / GIT -+++++++++++++++++ - -Common steps -================ - -.. index:: - double: installing; rename_configs - -.. note:: - The below operations should take place underneath your /etc folder. - -** Renaming configuration files is easy if you want to do it one by one then - do:: - $ mv bufunfa-central.conf.sample bufunfa-central.conf - -** You can also do it in one swoop:: - $ rename 's/\.sample$//' *.sample - - -Installing the Central -====================== - -.. index:: - double: installing; central - -1. Clone the bufunfa repo off of github:: - $ cd /opt/stack - $ git clone https://github.com/ekarlso/bufunfa.git - -2. As a user with ``root`` permissions or ``sudo`` privileges, run the - bufunfa installer:: - $ cd bufunfa - $ sudo python setup.py install - -3. See :rename_configs: - -4. Configure the :term:`central` service - - Change the wanted configuration settings to match your environment - :: - $ vi bufunfa-central.conf - - Refer to :doc:`configuration` details on configuring the service. - -5. Start the central service:: - $ bufunfa-central - - -Installing the Recorder -==================== - -.. index:: - double: installing; recorder - - -1. Clone the bufunfa repo off of github:: - $ cd /opt/stack - $ git clone https://github.com/ekarlso/bufunfa.git - -2. As a user with ``root`` permissions or ``sudo`` privileges, run the - bufunfa installer:: - $ cd bufunfa - $ sudo python setup.py install - -3. See :rename_configs: - -4. Configure the :term:`recorder` service - - Change the wanted configuration settings to match your environment - :: - $ vi bufunfa-recorder.conf - - Refer to :doc:`configuration` details on configuring the service. - -5. Start the Recorder service:: - $ bufunfa-recorder - - -Installing the API -==================== - -.. index:: - double: installing; api - -.. note:: - The API Server needs to able to talk to Keystone for AuthN + Z and - communicates via MQ to other services. - -1. Clone the bufunfa repo off of github:: - $ cd /opt/stack - $ git clone https://github.com/ekarlso/bufunfa.git - -2. As a user with ``root`` permissions or ``sudo`` privileges, run the - bufunfa installer:: - $ cd bufunfa - $ sudo python setup.py install - -3. See :rename_configs: - -4. Configure the :term:`api` service - - Change the wanted configuration settings to match your environment - :: - $ vi bufunfa-api.conf - $ vi bufunfa-api-paste.ini - - Refer to :doc:`configuration` details on configuring the service. - -5. Start the API service:: - $ bufunfa-api diff --git a/doc/source/services.dia b/doc/source/services.dia deleted file mode 100644 index 1504090..0000000 Binary files a/doc/source/services.dia and /dev/null differ diff --git a/doc/source/services.png b/doc/source/services.png deleted file mode 100644 index febcced..0000000 Binary files a/doc/source/services.png and /dev/null differ diff --git a/doc/source/storage_layout.dia b/doc/source/storage_layout.dia deleted file mode 100644 index 1951571..0000000 Binary files a/doc/source/storage_layout.dia and /dev/null differ diff --git a/doc/source/storage_layout.png b/doc/source/storage_layout.png deleted file mode 100644 index bb34006..0000000 Binary files a/doc/source/storage_layout.png and /dev/null differ diff --git a/openstack-common.conf b/openstack-common.conf deleted file mode 100644 index 0f3f23b..0000000 --- a/openstack-common.conf +++ /dev/null @@ -1,7 +0,0 @@ -[DEFAULT] - -# The list of modules to copy from openstack-common -modules=gettextutils,cfg,version,local,iniparser,utils,exception,timeutils,importutils,setup,log,jsonutils,notifier,rpc,excutils - -# The base module to hold the copy of openstack.common -base=bufunfa diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 549c26d..0000000 --- a/setup.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[nosetests] -cover-package=bufunfa -cover-html=true -cover-erase=true -cover-inclusive=true -verbosity=2 -detailed-errors=1 -where=bufunfa/tests - -[build_sphinx] -source-dir = doc/source -build-dir = doc/build -all_files = 1 - -[upload_docs] -upload-dir = doc/build/html diff --git a/setup.py b/setup.py deleted file mode 100755 index c8e7368..0000000 --- a/setup.py +++ /dev/null @@ -1,78 +0,0 @@ -#!onsusr/bin/env python -# Copyright 2012 Bouvet ASA -# -# Author: Endre Karlson -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from setuptools import setup, find_packages -import textwrap - -from bufunfa.openstack.common import setup as common_setup -from bufunfa.version import version_info as version - -install_requires = common_setup.parse_requirements(['tools/pip-requires']) -install_options = common_setup.parse_requirements(['tools/pip-options']) -tests_require = common_setup.parse_requirements(['tools/test-requires']) -setup_require = common_setup.parse_requirements(['tools/setup-requires']) -dependency_links = common_setup.parse_dependency_links([ - 'tools/pip-requires', - 'tools/pip-options', - 'tools/test-requires', - 'tools/setup-requires' -]) - -setup( - name='bufunfa', - version=version.canonical_version_string(always=True), - description='Billing as a Service', - author='Endre Karlson', - author_email='endre.karlson@bouvet.no', - url='https://launchpad.net/bufunfa', - packages=find_packages(exclude=['bin']), - include_package_data=True, - test_suite='nose.collector', - setup_requires=setup_require, - install_requires=install_requires, - tests_require=tests_require, - extras_require={ - 'test': tests_require, - 'optional': install_options, - }, - dependency_links=dependency_links, - scripts=[ - 'bin/bufunfa-api', - 'bin/bufunfa-central', - 'bin/bufunfa-sync', - 'bin/bufunfa-recorder' - ], - cmdclass=common_setup.get_cmdclass(), - entry_points=textwrap.dedent(""" - [bufunfa.storage] - mongodb = bufunfa.storage.impl_mongodb:MongoDBStorage - mysql = bufunfa.storage.impl_sqlalchemy:SQLAlchemyStorage - postgresql = bufunfa.storage.impl_sqlalchemy:SQLAlchemyStorage - sqlite = bufunfa.storage.impl_sqlalchemy:SQLAlchemyStorage - [bufunfa.recorder] - ceilometer = bufunfa.recorder.impl_ceilometer:RecordEngine - """), - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Topic :: Finance :: Billing Service', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Environment :: No Input/Output (Daemon)', - 'Environment :: OpenStack', - ] -) diff --git a/tools/pip-options b/tools/pip-options deleted file mode 100644 index 5c09a18..0000000 --- a/tools/pip-options +++ /dev/null @@ -1,9 +0,0 @@ -# Optional Stuff that is used by default -SQLAlchemy>=0.7.8,<=0.7.9 -kombu - -# Needed for Keystone Middleware -python-keystoneclient>=0.2.0 - -# Recorder - Ceilometer -http://github.com/dreamhost/ceilometerclient/archive/master.zip#egg=ceilometerclient diff --git a/tools/pip-requires b/tools/pip-requires deleted file mode 100644 index 0cbaeea..0000000 --- a/tools/pip-requires +++ /dev/null @@ -1,10 +0,0 @@ -Flask==0.9 -eventlet -jsonschema>=0.6 -PasteDeploy -stevedore - -# From OpenStack Common -routes>=1.12.3 -iso8601>=0.1.4 -WebOb>=1.0.8 diff --git a/tools/test-requires b/tools/test-requires deleted file mode 100644 index d7700fb..0000000 --- a/tools/test-requires +++ /dev/null @@ -1,7 +0,0 @@ -unittest2 -nose -openstack.nose_plugin -nosehtmloutput -mox -sphinx -sphinxcontrib-httpdomain diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 032791b..0000000 --- a/tox.ini +++ /dev/null @@ -1,39 +0,0 @@ -[tox] -envlist = py26,py27,pep8,pyflakes -minversion = 1.4.0 - -[tox:jenkins] -downloadcache = ~/cache/pip - -[testenv] -deps = -r{toxinidir}/tools/test-requires - -r{toxinidir}/tools/pip-options - -r{toxinidir}/tools/pip-requires -setenv = VIRTUAL_ENV={envdir} - NOSE_WITH_OPENSTACK=1 - NOSE_OPENSTACK_COLOR=1 - NOSE_OPENSTACK_RED=0.05 - NOSE_OPENSTACK_YELLOW=0.025 - NOSE_OPENSTACK_SHOW_ELAPSED=1 -commands = nosetests {posargs} -sitepackages = False - -[testenv:cover] -deps = {[testenv]deps} - coverage - nosexcover -setenv = {[testenv]setenv} - NOSE_WITH_COVERAGE=1 - -[testenv:pep8] -deps = {[testenv]deps} - pep8==1.3.3 -commands = pep8 --repeat --show-source --exclude=.venv,.tox,dist,doc,openstack bufunfa setup.py bin/bufunfa-api bin/bufunfa-central bin/bufunfa-recorder - -[testenv:pyflakes] -deps = {[testenv]deps} - pyflakes==0.5.0 -commands = pyflakes bufunfa bin setup.py - -[testenv:venv] -commands = {posargs}