diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index c8feb47..0000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[run] -branch = True -source = logaas -omit = logaas/openstack/* - -[report] -ignore-errors = True diff --git a/.gitignore b/.gitignore deleted file mode 100644 index d7fe34e..0000000 --- a/.gitignore +++ /dev/null @@ -1,39 +0,0 @@ -*.py[cod] - -# C extensions -*.so - -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -bin -var -sdist -develop-eggs -.installed.cfg -lib -lib64 - -# Installer logs -pip-log.txt - -# Unit test / coverage reports -.coverage -.tox -nosetests.xml -cover -.testrepository -*.sqlite -.venv - -# Translations -*.mo - -# Mr Developer -.mr.developer.cfg -.project -.pydevproject diff --git a/.gitreview b/.gitreview deleted file mode 100644 index b10e046..0000000 --- a/.gitreview +++ /dev/null @@ -1,4 +0,0 @@ -[gerrit] -host=review.openstack.org -port=29418 -project=stackforge/logaas.git diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index 2109af6..0000000 --- a/.testr.conf +++ /dev/null @@ -1,4 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 37ec93a..0000000 --- a/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.rst b/README.rst index eaa1bc5..9006052 100644 --- a/README.rst +++ b/README.rst @@ -1,38 +1,7 @@ -OpenStack Logging as a Service -============================== +This project is no longer maintained. -**Don't worry we are not going to reinvent logstash & syslog.** +The contents of this repository are still available in the Git source code +management system. To see the contents of this repository before it reached +its end of life, please check out the previous commit with +"git checkout HEAD^1". -Why one more service? ---------------------- - -As you probably know OpenStack is quite large ecosystem of -multiply services that are collaborating together to implement -OpenStack API. Sometimes to implement one OpenStack API method -you need to use multiply OpenStack services. It goes without -saying words that it will be hard to debug such well distributed -system, especially if you don't have logs. - -As we already known it's quite important to be able to store and -query log in OpenStack. But current OpenStack infrastructure won't -help you with this. - -Missing standard way to deal with logs produce to big issues: -1) Everybody that would like to work with OpenStack have to build -custom logging system (duplicating of efforts) -2) We are not able to provide OpenStack API for working with logs, -as a result we are not able to retrieve LOGs from Horizon. -3) We are not able to collabarate together to build a system of -aggregation & smart analyze of logs. - - - -Current goals -------------- - -#. Allows to setup :on presented resources logging service (e.g. logstash) -#. Autoscale of logging service -#. Querying logging serivce to retrive logs -#. Integration with keystone that will allow project to auto discover where to - store logs. No more billions changes in all `*.conf` files to setup logging - in all services diff --git a/babel.cfg b/babel.cfg deleted file mode 100644 index 1d15bb3..0000000 --- a/babel.cfg +++ /dev/null @@ -1 +0,0 @@ -[python: **.py] \ No newline at end of file diff --git a/doc/source/Makefile b/doc/source/Makefile deleted file mode 100644 index c9ad69f..0000000 --- a/doc/source/Makefile +++ /dev/null @@ -1,177 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/logaas.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/logaas.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/logaas" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/logaas" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/doc/source/conf.py b/doc/source/conf.py deleted file mode 100644 index d12cc88..0000000 --- a/doc/source/conf.py +++ /dev/null @@ -1,256 +0,0 @@ -# Logaas documentation build configuration file, created by -# sphinx-quickstart on Fri Jan 10 23:19:18 2014. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import datetime -import os -import sys - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.extend([ - os.path.abspath('../..'), -]) - - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', -] -todo_include_todos = True - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Logaas' -copyright = u'%d, Mirantis Inc.' % datetime.datetime.now().year - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '0.0.1' -# The full version, including alpha/beta/rc tags. -release = '0.0.1' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -git_cmd = "git log --pretty=format:'%ad, commit %h' --date=local -n1" -html_last_updated_fmt = os.popen(git_cmd).read() - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = '%sdoc' % project - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', - '%s.tex' % project, - u'%s Documentation' % project, - u'OpenStack Foundation', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -#man_pages = [ -# ('index', 'logaas', u'Logaas Documentation', -# [u'Logaas Team'], 1) -#] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'Logaas', u'Logaas Documentation', - u'Logaas Team', 'Logaas', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' diff --git a/doc/source/index.rst b/doc/source/index.rst deleted file mode 100644 index 65e915c..0000000 --- a/doc/source/index.rst +++ /dev/null @@ -1,71 +0,0 @@ -.. - Copyright 2014 Mirantis Inc. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Logging as a Service for OpenStack documentation -================================================ - -**Don't worry we are not going to reinvent logstash & syslog.** - -Why one more service? ---------------------- - -As you probably know OpenStack is quite large ecosystem of -multiply services that are collaborating together to implement -OpenStack API. Sometimes to implement one OpenStack API method -you need to use multiply OpenStack services. It goes without -saying words that it will be hard to debug such well distributed -system, especially if you don't have logs. - -As we already known it's quite important to be able to store and -query log in OpenStack. But current OpenStack infrastructure won't -help you with this. - -Missing standard way to deal with logs produce to big issues: -1) Everybody that would like to work with OpenStack have to build -custom logging system (duplicating of efforts) -2) We are not able to provide OpenStack API for working with logs, -as a result we are not able to retrieve LOGs from Horizon. -3) We are not able to collaborate together to build a system of -aggregation & smart analyze of logs. - - - -Current goals -------------- - -#. Allows to setup :on presented resources logging service (e.g. logstash) -#. Auto-scale of logging service -#. Querying logging service to retrieve logs -#. Integration with keystone that will allow project to auto discover where to - store logs. No more billions changes in all `*.conf` files to setup logging - in all services - - -Source Documentation -==================== - -Contents: - -.. toctree:: - :maxdepth: 3 - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/logaas/__init__.py b/logaas/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/logaas/cmd/__init__.py b/logaas/cmd/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/logaas/cmd/cliutils.py b/logaas/cmd/cliutils.py deleted file mode 100644 index 46703ac..0000000 --- a/logaas/cmd/cliutils.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2013: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import print_function - -import os -import sys - -from oslo.config import cfg - -from logaas.openstack.common.apiclient import exceptions -from logaas.openstack.common import cliutils -from logaas.openstack.common.db import options as db_options -from logaas.openstack.common.gettextutils import _ -from logaas.openstack.common import log as logging -from logaas import version - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - -# NOTE(hughsaunders): Added as default value is no longer stored in -# openstack.common.db.options, referenced in run() below. -# See: * https://review.openstack.org/84768 -# * https://bugs.launchpad.net/logaas/+bug/1302570 -_DEFAULT_SQL_CONNECTION = 'sqlite:////tmp/logaas.sqlite' - - -def pretty_float_formatter(field, ndigits=None): - """Create a formatter function for the given float field. - - :param field: a float object attribute name to be formatted. - :param ndigits: The number of digits after decimal point after round. - If None, then no rounding will be done. - :returns: the formatter function - """ - - def _formatter(obj): - value = getattr(obj, field) - if value is not None: - if ndigits is not None: - return round(value, ndigits) - else: - return value - else: - return "n/a" - return _formatter - - -def args(*args, **kwargs): - def _decorator(func): - func.__dict__.setdefault('args', []).insert(0, (args, kwargs)) - return func - return _decorator - - -def _methods_of(obj): - """Get all callable methods of an object that don't start with underscore - - returns a list of tuples of the form (method_name, method) - """ - result = [] - for i in dir(obj): - if callable(getattr(obj, i)) and not i.startswith('_'): - result.append((i, getattr(obj, i))) - return result - - -def _add_command_parsers(categories, subparsers): - parser = subparsers.add_parser('version') - - parser = subparsers.add_parser('bash-completion') - parser.add_argument('query_category', nargs='?') - - for category in categories: - command_object = categories[category]() - - parser = subparsers.add_parser(category) - parser.set_defaults(command_object=command_object) - - category_subparsers = parser.add_subparsers(dest='action') - - for (action, action_fn) in _methods_of(command_object): - parser = category_subparsers.add_parser(action) - - action_kwargs = [] - for args, kwargs in getattr(action_fn, 'args', []): - # FIXME(markmc): hack to assume dest is the arg name without - # the leading hyphens if no dest is supplied - kwargs.setdefault('dest', args[0][2:]) - if kwargs['dest'].startswith('action_kwarg_'): - action_kwargs.append(kwargs['dest'][len('action_kwarg_'):]) - else: - action_kwargs.append(kwargs['dest']) - kwargs['dest'] = 'action_kwarg_' + kwargs['dest'] - - parser.add_argument(*args, **kwargs) - - parser.set_defaults(action_fn=action_fn) - parser.set_defaults(action_kwargs=action_kwargs) - - parser.add_argument('action_args', nargs='*') - - -def run(argv, categories): - db_options.set_defaults(sql_connection=_DEFAULT_SQL_CONNECTION, - sqlite_db='logaas.sqlite') - parser = lambda subparsers: _add_command_parsers(categories, subparsers) - category_opt = cfg.SubCommandOpt('category', - title='Command categories', - help='Available categories', - handler=parser) - CONF.register_cli_opt(category_opt) - - try: - cfg.CONF(argv[1:], project='logaas', version=version.version_string()) - logging.setup("logaas") - if not CONF.get('log_config_append'): - # The below two lines are to disable noise from request module. The - # standard way should be we make such lots of settings on the root - # logaas. However current oslo code doesn't support such interface. - # So I choose to use a 'hacking' way to avoid INFO logs from - # request module where user didn't give specific log configuration. - # And we could remove this hacking after oslo.log has such - # interface. - LOG.debug("INFO logs from urllib3 and requests module are hide.") - requests_log = logging.getLogger("requests").logger - requests_log.setLevel(logging.logging.WARNING) - urllib3_log = logging.getLogger("urllib3").logger - urllib3_log.setLevel(logging.logging.WARNING) - except cfg.ConfigFilesNotFoundError: - cfgfile = CONF.config_file[-1] if CONF.config_file else None - if cfgfile and not os.access(cfgfile, os.R_OK): - st = os.stat(cfgfile) - print(_("Could not read %s. Re-running with sudo") % cfgfile) - try: - os.execvp('sudo', ['sudo', '-u', '#%s' % st.st_uid] + sys.argv) - except Exception: - print(_('sudo failed, continuing as if nothing happened')) - - print(_('Please re-run %s as root.') % argv[0]) - return(2) - - if CONF.category.name == "version": - print(version.version_string()) - return(0) - - if CONF.category.name == "bash-completion": - if not CONF.category.query_category: - print(" ".join(categories.keys())) - elif CONF.category.query_category in categories: - fn = categories[CONF.category.query_category] - command_object = fn() - actions = _methods_of(command_object) - print(" ".join([k for (k, v) in actions])) - return(0) - - fn = CONF.category.action_fn - fn_args = [arg.decode('utf-8') for arg in CONF.category.action_args] - fn_kwargs = {} - for k in CONF.category.action_kwargs: - v = getattr(CONF.category, 'action_kwarg_' + k) - if v is None: - continue - if isinstance(v, basestring): - v = v.decode('utf-8') - fn_kwargs[k] = v - - # call the action with the remaining arguments - # check arguments - try: - cliutils.validate_args(fn, *fn_args, **fn_kwargs) - except exceptions.MissingArgs as e: - # NOTE(mikal): this isn't the most helpful error message ever. It is - # long, and tells you a lot of things you probably don't want to know - # if you just got a single arg wrong. - print(fn.__doc__) - CONF.print_help() - print(e) - return(1) - try: - ret = fn(*fn_args, **fn_kwargs) - return(ret) - except Exception: - print(_("Command failed, please check log for more info")) - raise diff --git a/logaas/cmd/manage.py b/logaas/cmd/manage.py deleted file mode 100644 index 1cead8d..0000000 --- a/logaas/cmd/manage.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""CLI for LogaaS management.""" - -from __future__ import print_function - -import sys - -from logaas.cmd import cliutils -from logaas import db - - -class DBCommands(object): - """Commands for DB management.""" - - def create(self): - """Create database for logaas.""" - db.db_create() - - def recreate(self): - """Drop and then create database for logaas.""" - db.db_drop() - db.db_create() - - -def main(): - categories = {'db': DBCommands} - cliutils.run(sys.argv, categories) - - -if __name__ == '__main__': - main() diff --git a/logaas/db/__init__.py b/logaas/db/__init__.py deleted file mode 100644 index 60f6805..0000000 --- a/logaas/db/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from logaas.db.api import * # noqa diff --git a/logaas/db/api.py b/logaas/db/api.py deleted file mode 100644 index 32345b7..0000000 --- a/logaas/db/api.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Defines interface for DB access. - -The underlying driver is loaded as a :class:`LazyPluggable`. - -Functions in this module are imported into the logaas.db namespace. Call these -functions from logaas.db namespace, not the logaas.db.api namespace. - -All functions in this module return objects that implement a dictionary-like -interface. Currently, many of these objects are sqlalchemy objects that -implement a dictionary interface. However, a future goal is to have all of -these objects be simple dictionaries. - - -**Related Flags** - -:backend: string to lookup in the list of LazyPluggable backends. - `sqlalchemy` is the only supported backend right now. - -:connection: string specifying the sqlalchemy connection to use, like: - `sqlite:///var/lib/logaas/logaas.sqlite`. - -:enable_new_services: when adding a new service to the database, is it in the - pool of available hardware (Default: True) - -""" - -from oslo.config import cfg - -from logaas.openstack.common.db import api as db_api - - -CONF = cfg.CONF - - -CONF.import_opt('backend', 'logaas.openstack.common.db.options', - group='database') - - -_BACKEND_MAPPING = {'sqlalchemy': 'logaas.db.sqlalchemy.api'} - -IMPL = db_api.DBAPI(CONF.database.backend, backend_mapping=_BACKEND_MAPPING) - - -def db_cleanup(): - """Recreate engine.""" - IMPL.db_cleanup() - - -def db_create(): - """Initialize DB. This method will drop existing database.""" - IMPL.db_create() - - -def db_drop(): - """Drop DB. This method drop existing database.""" - IMPL.db_drop() diff --git a/logaas/db/sqlalchemy/__init__.py b/logaas/db/sqlalchemy/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/logaas/db/sqlalchemy/api.py b/logaas/db/sqlalchemy/api.py deleted file mode 100644 index 8208830..0000000 --- a/logaas/db/sqlalchemy/api.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -SQLAlchemy implementation for DB.API -""" - -from oslo.config import cfg - -from logaas.db.sqlalchemy import models -from logaas.openstack.common.db.sqlalchemy import session as db_session -from logaas.openstack.common.gettextutils import _ - - -CONF = cfg.CONF - -CONF.import_opt('connection', - 'logaas.openstack.common.db.options', - group='database') - -_FACADE = None - - -def _create_facade_lazily(): - global _FACADE - - if _FACADE is None: - _FACADE = db_session.EngineFacade.from_config( - CONF.database.connection, CONF) - - return _FACADE - - -def get_engine(): - facade = _create_facade_lazily() - return facade.get_engine() - - -def get_session(**kwargs): - facade = _create_facade_lazily() - return facade.get_session(**kwargs) - - -def get_backend(): - """The backend is this module itself.""" - return Connection() - - -class Connection(object): - - def db_cleanup(self): - global _FACADE - - _FACADE = None - - def db_create(self): - models.create_db() - - def db_drop(self): - models.drop_db() - - def model_query(self, model, session=None): - """The helper method to create query. - - :param model: The instance of - :class:`logaas.db.sqlalchemy.models.LogaasBase` to - request it. - :param session: Reuse the session object or get new one if it is - None. - :returns: The query object. - :raises: :class:`Exception` when the model is not a subclass of - :class:`logaas.db.sqlalchemy.models.RallyBase`. - """ - session = session or get_session() - query = session.query(model) - - def issubclassof_logaas_base(obj): - return isinstance(obj, type) and issubclass(obj, models.LogaasBase) - - if not issubclassof_logaas_base(model): - raise Exception(_("The model should be a subclass of LogaasBase")) - - return query diff --git a/logaas/db/sqlalchemy/models.py b/logaas/db/sqlalchemy/models.py deleted file mode 100644 index 39b9616..0000000 --- a/logaas/db/sqlalchemy/models.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -SQLAlchemy models for logaas data. -""" -import uuid - -from sqlalchemy.ext.declarative import declarative_base - -from logaas.openstack.common.db.sqlalchemy import models - - -BASE = declarative_base() - - -def UUID(): - return str(uuid.uuid4()) - - -class LogaasBase(models.TimestampMixin, - models.ModelBase): - metadata = None - - def save(self, session=None): - from logaas.db.sqlalchemy import api as sa_api - - if session is None: - session = sa_api.get_session() - - super(LogaasBase, self).save(session=session) - - -def create_db(): - from logaas.db.sqlalchemy import api as sa_api - - BASE.metadata.create_all(sa_api.get_engine()) - - -def drop_db(): - from logaas.db.sqlalchemy import api as sa_api - - engine = sa_api.get_engine() - OLD_BASE = declarative_base() - OLD_BASE.metadata.reflect(bind=engine) - OLD_BASE.metadata.drop_all(engine, checkfirst=True) diff --git a/logaas/exceptions.py b/logaas/exceptions.py deleted file mode 100644 index ee1b6c3..0000000 --- a/logaas/exceptions.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sys - -from oslo.config import cfg - -from logaas.openstack.common.gettextutils import _ -from logaas.openstack.common import log as logging - -LOG = logging.getLogger(__name__) - -exc_log_opts = [ - cfg.BoolOpt('fatal_exception_format_errors', - default=False, - help='make exception message format errors fatal'), -] - -CONF = cfg.CONF -CONF.register_opts(exc_log_opts) - - -class LogaasException(Exception): - """Base Logaas Exception - - To correctly use this class, inherit from it and define - a 'msg_fmt' property. That msg_fmt will get printf'd - with the keyword arguments provided to the constructor. - - """ - msg_fmt = _("An unknown exception occurred.") - - def __init__(self, message=None, **kwargs): - self.kwargs = kwargs - - if 'code' not in self.kwargs: - try: - self.kwargs['code'] = self.code - except AttributeError: - pass - - if not message: - try: - message = self.msg_fmt % kwargs - except KeyError: - exc_info = sys.exc_info() - # kwargs doesn't match a variable in the message - # log the issue and the kwargs - msg = "kwargs don't match in string format operation: %s" - LOG.debug(msg % kwargs, exc_info=exc_info) - - if CONF.fatal_exception_format_errors: - raise exc_info[0], exc_info[1], exc_info[2] - else: - # at least get the core message out if something happened - message = self.msg_fmt - - super(LogaasException, self).__init__(message) - - def format_message(self): - if self.__class__.__name__.endswith('_Remote'): - return self.args[0] - else: - return unicode(self) diff --git a/logaas/openstack/__init__.py b/logaas/openstack/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/logaas/openstack/common/__init__.py b/logaas/openstack/common/__init__.py deleted file mode 100644 index d1223ea..0000000 --- a/logaas/openstack/common/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import six - - -six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox')) diff --git a/logaas/openstack/common/apiclient/__init__.py b/logaas/openstack/common/apiclient/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/logaas/openstack/common/apiclient/auth.py b/logaas/openstack/common/apiclient/auth.py deleted file mode 100644 index a9e14c4..0000000 --- a/logaas/openstack/common/apiclient/auth.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# Copyright 2013 Spanish National Research Council. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# E0202: An attribute inherited from %s hide this method -# pylint: disable=E0202 - -import abc -import argparse -import os - -import six -from stevedore import extension - -from logaas.openstack.common.apiclient import exceptions - - -_discovered_plugins = {} - - -def discover_auth_systems(): - """Discover the available auth-systems. - - This won't take into account the old style auth-systems. - """ - global _discovered_plugins - _discovered_plugins = {} - - def add_plugin(ext): - _discovered_plugins[ext.name] = ext.plugin - - ep_namespace = "logaas.openstack.common.apiclient.auth" - mgr = extension.ExtensionManager(ep_namespace) - mgr.map(add_plugin) - - -def load_auth_system_opts(parser): - """Load options needed by the available auth-systems into a parser. - - This function will try to populate the parser with options from the - available plugins. - """ - group = parser.add_argument_group("Common auth options") - BaseAuthPlugin.add_common_opts(group) - for name, auth_plugin in six.iteritems(_discovered_plugins): - group = parser.add_argument_group( - "Auth-system '%s' options" % name, - conflict_handler="resolve") - auth_plugin.add_opts(group) - - -def load_plugin(auth_system): - try: - plugin_class = _discovered_plugins[auth_system] - except KeyError: - raise exceptions.AuthSystemNotFound(auth_system) - return plugin_class(auth_system=auth_system) - - -def load_plugin_from_args(args): - """Load required plugin and populate it with options. - - Try to guess auth system if it is not specified. Systems are tried in - alphabetical order. - - :type args: argparse.Namespace - :raises: AuthPluginOptionsMissing - """ - auth_system = args.os_auth_system - if auth_system: - plugin = load_plugin(auth_system) - plugin.parse_opts(args) - plugin.sufficient_options() - return plugin - - for plugin_auth_system in sorted(six.iterkeys(_discovered_plugins)): - plugin_class = _discovered_plugins[plugin_auth_system] - plugin = plugin_class() - plugin.parse_opts(args) - try: - plugin.sufficient_options() - except exceptions.AuthPluginOptionsMissing: - continue - return plugin - raise exceptions.AuthPluginOptionsMissing(["auth_system"]) - - -@six.add_metaclass(abc.ABCMeta) -class BaseAuthPlugin(object): - """Base class for authentication plugins. - - An authentication plugin needs to override at least the authenticate - method to be a valid plugin. - """ - - auth_system = None - opt_names = [] - common_opt_names = [ - "auth_system", - "username", - "password", - "tenant_name", - "token", - "auth_url", - ] - - def __init__(self, auth_system=None, **kwargs): - self.auth_system = auth_system or self.auth_system - self.opts = dict((name, kwargs.get(name)) - for name in self.opt_names) - - @staticmethod - def _parser_add_opt(parser, opt): - """Add an option to parser in two variants. - - :param opt: option name (with underscores) - """ - dashed_opt = opt.replace("_", "-") - env_var = "OS_%s" % opt.upper() - arg_default = os.environ.get(env_var, "") - arg_help = "Defaults to env[%s]." % env_var - parser.add_argument( - "--os-%s" % dashed_opt, - metavar="<%s>" % dashed_opt, - default=arg_default, - help=arg_help) - parser.add_argument( - "--os_%s" % opt, - metavar="<%s>" % dashed_opt, - help=argparse.SUPPRESS) - - @classmethod - def add_opts(cls, parser): - """Populate the parser with the options for this plugin. - """ - for opt in cls.opt_names: - # use `BaseAuthPlugin.common_opt_names` since it is never - # changed in child classes - if opt not in BaseAuthPlugin.common_opt_names: - cls._parser_add_opt(parser, opt) - - @classmethod - def add_common_opts(cls, parser): - """Add options that are common for several plugins. - """ - for opt in cls.common_opt_names: - cls._parser_add_opt(parser, opt) - - @staticmethod - def get_opt(opt_name, args): - """Return option name and value. - - :param opt_name: name of the option, e.g., "username" - :param args: parsed arguments - """ - return (opt_name, getattr(args, "os_%s" % opt_name, None)) - - def parse_opts(self, args): - """Parse the actual auth-system options if any. - - This method is expected to populate the attribute `self.opts` with a - dict containing the options and values needed to make authentication. - """ - self.opts.update(dict(self.get_opt(opt_name, args) - for opt_name in self.opt_names)) - - def authenticate(self, http_client): - """Authenticate using plugin defined method. - - The method usually analyses `self.opts` and performs - a request to authentication server. - - :param http_client: client object that needs authentication - :type http_client: HTTPClient - :raises: AuthorizationFailure - """ - self.sufficient_options() - self._do_authenticate(http_client) - - @abc.abstractmethod - def _do_authenticate(self, http_client): - """Protected method for authentication. - """ - - def sufficient_options(self): - """Check if all required options are present. - - :raises: AuthPluginOptionsMissing - """ - missing = [opt - for opt in self.opt_names - if not self.opts.get(opt)] - if missing: - raise exceptions.AuthPluginOptionsMissing(missing) - - @abc.abstractmethod - def token_and_endpoint(self, endpoint_type, service_type): - """Return token and endpoint. - - :param service_type: Service type of the endpoint - :type service_type: string - :param endpoint_type: Type of endpoint. - Possible values: public or publicURL, - internal or internalURL, - admin or adminURL - :type endpoint_type: string - :returns: tuple of token and endpoint strings - :raises: EndpointException - """ diff --git a/logaas/openstack/common/apiclient/base.py b/logaas/openstack/common/apiclient/base.py deleted file mode 100644 index 2ecc7b5..0000000 --- a/logaas/openstack/common/apiclient/base.py +++ /dev/null @@ -1,507 +0,0 @@ -# Copyright 2010 Jacob Kaplan-Moss -# Copyright 2011 OpenStack Foundation -# Copyright 2012 Grid Dynamics -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Base utilities to build API operation managers and objects on top of. -""" - -# E1102: %s is not callable -# pylint: disable=E1102 - -import abc -import copy - -import six -from six.moves.urllib import parse - -from logaas.openstack.common.apiclient import exceptions -from logaas.openstack.common.gettextutils import _ -from logaas.openstack.common import strutils - - -def getid(obj): - """Return id if argument is a Resource. - - Abstracts the common pattern of allowing both an object or an object's ID - (UUID) as a parameter when dealing with relationships. - """ - try: - if obj.uuid: - return obj.uuid - except AttributeError: - pass - try: - return obj.id - except AttributeError: - return obj - - -# TODO(aababilov): call run_hooks() in HookableMixin's child classes -class HookableMixin(object): - """Mixin so classes can register and run hooks.""" - _hooks_map = {} - - @classmethod - def add_hook(cls, hook_type, hook_func): - """Add a new hook of specified type. - - :param cls: class that registers hooks - :param hook_type: hook type, e.g., '__pre_parse_args__' - :param hook_func: hook function - """ - if hook_type not in cls._hooks_map: - cls._hooks_map[hook_type] = [] - - cls._hooks_map[hook_type].append(hook_func) - - @classmethod - def run_hooks(cls, hook_type, *args, **kwargs): - """Run all hooks of specified type. - - :param cls: class that registers hooks - :param hook_type: hook type, e.g., '__pre_parse_args__' - :param args: args to be passed to every hook function - :param kwargs: kwargs to be passed to every hook function - """ - hook_funcs = cls._hooks_map.get(hook_type) or [] - for hook_func in hook_funcs: - hook_func(*args, **kwargs) - - -class BaseManager(HookableMixin): - """Basic manager type providing common operations. - - Managers interact with a particular type of API (servers, flavors, images, - etc.) and provide CRUD operations for them. - """ - resource_class = None - - def __init__(self, client): - """Initializes BaseManager with `client`. - - :param client: instance of BaseClient descendant for HTTP requests - """ - super(BaseManager, self).__init__() - self.client = client - - def _list(self, url, response_key, obj_class=None, json=None): - """List the collection. - - :param url: a partial URL, e.g., '/servers' - :param response_key: the key to be looked up in response dictionary, - e.g., 'servers' - :param obj_class: class for constructing the returned objects - (self.resource_class will be used by default) - :param json: data that will be encoded as JSON and passed in POST - request (GET will be sent by default) - """ - if json: - body = self.client.post(url, json=json).json() - else: - body = self.client.get(url).json() - - if obj_class is None: - obj_class = self.resource_class - - data = body[response_key] - # NOTE(ja): keystone returns values as list as {'values': [ ... ]} - # unlike other services which just return the list... - try: - data = data['values'] - except (KeyError, TypeError): - pass - - return [obj_class(self, res, loaded=True) for res in data if res] - - def _get(self, url, response_key): - """Get an object from collection. - - :param url: a partial URL, e.g., '/servers' - :param response_key: the key to be looked up in response dictionary, - e.g., 'server' - """ - body = self.client.get(url).json() - return self.resource_class(self, body[response_key], loaded=True) - - def _head(self, url): - """Retrieve request headers for an object. - - :param url: a partial URL, e.g., '/servers' - """ - resp = self.client.head(url) - return resp.status_code == 204 - - def _post(self, url, json, response_key, return_raw=False): - """Create an object. - - :param url: a partial URL, e.g., '/servers' - :param json: data that will be encoded as JSON and passed in POST - request (GET will be sent by default) - :param response_key: the key to be looked up in response dictionary, - e.g., 'servers' - :param return_raw: flag to force returning raw JSON instead of - Python object of self.resource_class - """ - body = self.client.post(url, json=json).json() - if return_raw: - return body[response_key] - return self.resource_class(self, body[response_key]) - - def _put(self, url, json=None, response_key=None): - """Update an object with PUT method. - - :param url: a partial URL, e.g., '/servers' - :param json: data that will be encoded as JSON and passed in POST - request (GET will be sent by default) - :param response_key: the key to be looked up in response dictionary, - e.g., 'servers' - """ - resp = self.client.put(url, json=json) - # PUT requests may not return a body - if resp.content: - body = resp.json() - if response_key is not None: - return self.resource_class(self, body[response_key]) - else: - return self.resource_class(self, body) - - def _patch(self, url, json=None, response_key=None): - """Update an object with PATCH method. - - :param url: a partial URL, e.g., '/servers' - :param json: data that will be encoded as JSON and passed in POST - request (GET will be sent by default) - :param response_key: the key to be looked up in response dictionary, - e.g., 'servers' - """ - body = self.client.patch(url, json=json).json() - if response_key is not None: - return self.resource_class(self, body[response_key]) - else: - return self.resource_class(self, body) - - def _delete(self, url): - """Delete an object. - - :param url: a partial URL, e.g., '/servers/my-server' - """ - return self.client.delete(url) - - -@six.add_metaclass(abc.ABCMeta) -class ManagerWithFind(BaseManager): - """Manager with additional `find()`/`findall()` methods.""" - - @abc.abstractmethod - def list(self): - pass - - def find(self, **kwargs): - """Find a single item with attributes matching ``**kwargs``. - - This isn't very efficient: it loads the entire list then filters on - the Python side. - """ - matches = self.findall(**kwargs) - num_matches = len(matches) - if num_matches == 0: - msg = _("No %(name)s matching %(args)s.") % { - 'name': self.resource_class.__name__, - 'args': kwargs - } - raise exceptions.NotFound(msg) - elif num_matches > 1: - raise exceptions.NoUniqueMatch() - else: - return matches[0] - - def findall(self, **kwargs): - """Find all items with attributes matching ``**kwargs``. - - This isn't very efficient: it loads the entire list then filters on - the Python side. - """ - found = [] - searches = kwargs.items() - - for obj in self.list(): - try: - if all(getattr(obj, attr) == value - for (attr, value) in searches): - found.append(obj) - except AttributeError: - continue - - return found - - -class CrudManager(BaseManager): - """Base manager class for manipulating entities. - - Children of this class are expected to define a `collection_key` and `key`. - - - `collection_key`: Usually a plural noun by convention (e.g. `entities`); - used to refer collections in both URL's (e.g. `/v3/entities`) and JSON - objects containing a list of member resources (e.g. `{'entities': [{}, - {}, {}]}`). - - `key`: Usually a singular noun by convention (e.g. `entity`); used to - refer to an individual member of the collection. - - """ - collection_key = None - key = None - - def build_url(self, base_url=None, **kwargs): - """Builds a resource URL for the given kwargs. - - Given an example collection where `collection_key = 'entities'` and - `key = 'entity'`, the following URL's could be generated. - - By default, the URL will represent a collection of entities, e.g.:: - - /entities - - If kwargs contains an `entity_id`, then the URL will represent a - specific member, e.g.:: - - /entities/{entity_id} - - :param base_url: if provided, the generated URL will be appended to it - """ - url = base_url if base_url is not None else '' - - url += '/%s' % self.collection_key - - # do we have a specific entity? - entity_id = kwargs.get('%s_id' % self.key) - if entity_id is not None: - url += '/%s' % entity_id - - return url - - def _filter_kwargs(self, kwargs): - """Drop null values and handle ids.""" - for key, ref in six.iteritems(kwargs.copy()): - if ref is None: - kwargs.pop(key) - else: - if isinstance(ref, Resource): - kwargs.pop(key) - kwargs['%s_id' % key] = getid(ref) - return kwargs - - def create(self, **kwargs): - kwargs = self._filter_kwargs(kwargs) - return self._post( - self.build_url(**kwargs), - {self.key: kwargs}, - self.key) - - def get(self, **kwargs): - kwargs = self._filter_kwargs(kwargs) - return self._get( - self.build_url(**kwargs), - self.key) - - def head(self, **kwargs): - kwargs = self._filter_kwargs(kwargs) - return self._head(self.build_url(**kwargs)) - - def list(self, base_url=None, **kwargs): - """List the collection. - - :param base_url: if provided, the generated URL will be appended to it - """ - kwargs = self._filter_kwargs(kwargs) - - return self._list( - '%(base_url)s%(query)s' % { - 'base_url': self.build_url(base_url=base_url, **kwargs), - 'query': '?%s' % parse.urlencode(kwargs) if kwargs else '', - }, - self.collection_key) - - def put(self, base_url=None, **kwargs): - """Update an element. - - :param base_url: if provided, the generated URL will be appended to it - """ - kwargs = self._filter_kwargs(kwargs) - - return self._put(self.build_url(base_url=base_url, **kwargs)) - - def update(self, **kwargs): - kwargs = self._filter_kwargs(kwargs) - params = kwargs.copy() - params.pop('%s_id' % self.key) - - return self._patch( - self.build_url(**kwargs), - {self.key: params}, - self.key) - - def delete(self, **kwargs): - kwargs = self._filter_kwargs(kwargs) - - return self._delete( - self.build_url(**kwargs)) - - def find(self, base_url=None, **kwargs): - """Find a single item with attributes matching ``**kwargs``. - - :param base_url: if provided, the generated URL will be appended to it - """ - kwargs = self._filter_kwargs(kwargs) - - rl = self._list( - '%(base_url)s%(query)s' % { - 'base_url': self.build_url(base_url=base_url, **kwargs), - 'query': '?%s' % parse.urlencode(kwargs) if kwargs else '', - }, - self.collection_key) - num = len(rl) - - if num == 0: - msg = _("No %(name)s matching %(args)s.") % { - 'name': self.resource_class.__name__, - 'args': kwargs - } - raise exceptions.NotFound(404, msg) - elif num > 1: - raise exceptions.NoUniqueMatch - else: - return rl[0] - - -class Extension(HookableMixin): - """Extension descriptor.""" - - SUPPORTED_HOOKS = ('__pre_parse_args__', '__post_parse_args__') - manager_class = None - - def __init__(self, name, module): - super(Extension, self).__init__() - self.name = name - self.module = module - self._parse_extension_module() - - def _parse_extension_module(self): - self.manager_class = None - for attr_name, attr_value in self.module.__dict__.items(): - if attr_name in self.SUPPORTED_HOOKS: - self.add_hook(attr_name, attr_value) - else: - try: - if issubclass(attr_value, BaseManager): - self.manager_class = attr_value - except TypeError: - pass - - def __repr__(self): - return "" % self.name - - -class Resource(object): - """Base class for OpenStack resources (tenant, user, etc.). - - This is pretty much just a bag for attributes. - """ - - HUMAN_ID = False - NAME_ATTR = 'name' - - def __init__(self, manager, info, loaded=False): - """Populate and bind to a manager. - - :param manager: BaseManager object - :param info: dictionary representing resource attributes - :param loaded: prevent lazy-loading if set to True - """ - self.manager = manager - self._info = info - self._add_details(info) - self._loaded = loaded - - def __repr__(self): - reprkeys = sorted(k - for k in self.__dict__.keys() - if k[0] != '_' and k != 'manager') - info = ", ".join("%s=%s" % (k, getattr(self, k)) for k in reprkeys) - return "<%s %s>" % (self.__class__.__name__, info) - - @property - def human_id(self): - """Human-readable ID which can be used for bash completion. - """ - if self.NAME_ATTR in self.__dict__ and self.HUMAN_ID: - return strutils.to_slug(getattr(self, self.NAME_ATTR)) - return None - - def _add_details(self, info): - for (k, v) in six.iteritems(info): - try: - setattr(self, k, v) - self._info[k] = v - except AttributeError: - # In this case we already defined the attribute on the class - pass - - def __getattr__(self, k): - if k not in self.__dict__: - #NOTE(bcwaldon): disallow lazy-loading if already loaded once - if not self.is_loaded(): - self.get() - return self.__getattr__(k) - - raise AttributeError(k) - else: - return self.__dict__[k] - - def get(self): - """Support for lazy loading details. - - Some clients, such as novaclient have the option to lazy load the - details, details which can be loaded with this function. - """ - # set_loaded() first ... so if we have to bail, we know we tried. - self.set_loaded(True) - if not hasattr(self.manager, 'get'): - return - - new = self.manager.get(self.id) - if new: - self._add_details(new._info) - - def __eq__(self, other): - if not isinstance(other, Resource): - return NotImplemented - # two resources of different types are not equal - if not isinstance(other, self.__class__): - return False - if hasattr(self, 'id') and hasattr(other, 'id'): - return self.id == other.id - return self._info == other._info - - def is_loaded(self): - return self._loaded - - def set_loaded(self, val): - self._loaded = val - - def to_dict(self): - return copy.deepcopy(self._info) diff --git a/logaas/openstack/common/apiclient/client.py b/logaas/openstack/common/apiclient/client.py deleted file mode 100644 index c225973..0000000 --- a/logaas/openstack/common/apiclient/client.py +++ /dev/null @@ -1,364 +0,0 @@ -# Copyright 2010 Jacob Kaplan-Moss -# Copyright 2011 OpenStack Foundation -# Copyright 2011 Piston Cloud Computing, Inc. -# Copyright 2013 Alessio Ababilov -# Copyright 2013 Grid Dynamics -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -OpenStack Client interface. Handles the REST calls and responses. -""" - -# E0202: An attribute inherited from %s hide this method -# pylint: disable=E0202 - -import logging -import time - -try: - import simplejson as json -except ImportError: - import json - -import requests - -from logaas.openstack.common.apiclient import exceptions -from logaas.openstack.common.gettextutils import _ -from logaas.openstack.common import importutils - - -_logger = logging.getLogger(__name__) - - -class HTTPClient(object): - """This client handles sending HTTP requests to OpenStack servers. - - Features: - - - share authentication information between several clients to different - services (e.g., for compute and image clients); - - reissue authentication request for expired tokens; - - encode/decode JSON bodies; - - raise exceptions on HTTP errors; - - pluggable authentication; - - store authentication information in a keyring; - - store time spent for requests; - - register clients for particular services, so one can use - `http_client.identity` or `http_client.compute`; - - log requests and responses in a format that is easy to copy-and-paste - into terminal and send the same request with curl. - """ - - user_agent = "logaas.openstack.common.apiclient" - - def __init__(self, - auth_plugin, - region_name=None, - endpoint_type="publicURL", - original_ip=None, - verify=True, - cert=None, - timeout=None, - timings=False, - keyring_saver=None, - debug=False, - user_agent=None, - http=None): - self.auth_plugin = auth_plugin - - self.endpoint_type = endpoint_type - self.region_name = region_name - - self.original_ip = original_ip - self.timeout = timeout - self.verify = verify - self.cert = cert - - self.keyring_saver = keyring_saver - self.debug = debug - self.user_agent = user_agent or self.user_agent - - self.times = [] # [("item", starttime, endtime), ...] - self.timings = timings - - # requests within the same session can reuse TCP connections from pool - self.http = http or requests.Session() - - self.cached_token = None - - def _http_log_req(self, method, url, kwargs): - if not self.debug: - return - - string_parts = [ - "curl -i", - "-X '%s'" % method, - "'%s'" % url, - ] - - for element in kwargs['headers']: - header = "-H '%s: %s'" % (element, kwargs['headers'][element]) - string_parts.append(header) - - _logger.debug("REQ: %s" % " ".join(string_parts)) - if 'data' in kwargs: - _logger.debug("REQ BODY: %s\n" % (kwargs['data'])) - - def _http_log_resp(self, resp): - if not self.debug: - return - _logger.debug( - "RESP: [%s] %s\n", - resp.status_code, - resp.headers) - if resp._content_consumed: - _logger.debug( - "RESP BODY: %s\n", - resp.text) - - def serialize(self, kwargs): - if kwargs.get('json') is not None: - kwargs['headers']['Content-Type'] = 'application/json' - kwargs['data'] = json.dumps(kwargs['json']) - try: - del kwargs['json'] - except KeyError: - pass - - def get_timings(self): - return self.times - - def reset_timings(self): - self.times = [] - - def request(self, method, url, **kwargs): - """Send an http request with the specified characteristics. - - Wrapper around `requests.Session.request` to handle tasks such as - setting headers, JSON encoding/decoding, and error handling. - - :param method: method of HTTP request - :param url: URL of HTTP request - :param kwargs: any other parameter that can be passed to - requests.Session.request (such as `headers`) or `json` - that will be encoded as JSON and used as `data` argument - """ - kwargs.setdefault("headers", kwargs.get("headers", {})) - kwargs["headers"]["User-Agent"] = self.user_agent - if self.original_ip: - kwargs["headers"]["Forwarded"] = "for=%s;by=%s" % ( - self.original_ip, self.user_agent) - if self.timeout is not None: - kwargs.setdefault("timeout", self.timeout) - kwargs.setdefault("verify", self.verify) - if self.cert is not None: - kwargs.setdefault("cert", self.cert) - self.serialize(kwargs) - - self._http_log_req(method, url, kwargs) - if self.timings: - start_time = time.time() - resp = self.http.request(method, url, **kwargs) - if self.timings: - self.times.append(("%s %s" % (method, url), - start_time, time.time())) - self._http_log_resp(resp) - - if resp.status_code >= 400: - _logger.debug( - "Request returned failure status: %s", - resp.status_code) - raise exceptions.from_response(resp, method, url) - - return resp - - @staticmethod - def concat_url(endpoint, url): - """Concatenate endpoint and final URL. - - E.g., "http://keystone/v2.0/" and "/tokens" are concatenated to - "http://keystone/v2.0/tokens". - - :param endpoint: the base URL - :param url: the final URL - """ - return "%s/%s" % (endpoint.rstrip("/"), url.strip("/")) - - def client_request(self, client, method, url, **kwargs): - """Send an http request using `client`'s endpoint and specified `url`. - - If request was rejected as unauthorized (possibly because the token is - expired), issue one authorization attempt and send the request once - again. - - :param client: instance of BaseClient descendant - :param method: method of HTTP request - :param url: URL of HTTP request - :param kwargs: any other parameter that can be passed to - `HTTPClient.request` - """ - - filter_args = { - "endpoint_type": client.endpoint_type or self.endpoint_type, - "service_type": client.service_type, - } - token, endpoint = (self.cached_token, client.cached_endpoint) - just_authenticated = False - if not (token and endpoint): - try: - token, endpoint = self.auth_plugin.token_and_endpoint( - **filter_args) - except exceptions.EndpointException: - pass - if not (token and endpoint): - self.authenticate() - just_authenticated = True - token, endpoint = self.auth_plugin.token_and_endpoint( - **filter_args) - if not (token and endpoint): - raise exceptions.AuthorizationFailure( - _("Cannot find endpoint or token for request")) - - old_token_endpoint = (token, endpoint) - kwargs.setdefault("headers", {})["X-Auth-Token"] = token - self.cached_token = token - client.cached_endpoint = endpoint - # Perform the request once. If we get Unauthorized, then it - # might be because the auth token expired, so try to - # re-authenticate and try again. If it still fails, bail. - try: - return self.request( - method, self.concat_url(endpoint, url), **kwargs) - except exceptions.Unauthorized as unauth_ex: - if just_authenticated: - raise - self.cached_token = None - client.cached_endpoint = None - self.authenticate() - try: - token, endpoint = self.auth_plugin.token_and_endpoint( - **filter_args) - except exceptions.EndpointException: - raise unauth_ex - if (not (token and endpoint) or - old_token_endpoint == (token, endpoint)): - raise unauth_ex - self.cached_token = token - client.cached_endpoint = endpoint - kwargs["headers"]["X-Auth-Token"] = token - return self.request( - method, self.concat_url(endpoint, url), **kwargs) - - def add_client(self, base_client_instance): - """Add a new instance of :class:`BaseClient` descendant. - - `self` will store a reference to `base_client_instance`. - - Example: - - >>> def test_clients(): - ... from keystoneclient.auth import keystone - ... from openstack.common.apiclient import client - ... auth = keystone.KeystoneAuthPlugin( - ... username="user", password="pass", tenant_name="tenant", - ... auth_url="http://auth:5000/v2.0") - ... openstack_client = client.HTTPClient(auth) - ... # create nova client - ... from novaclient.v1_1 import client - ... client.Client(openstack_client) - ... # create keystone client - ... from keystoneclient.v2_0 import client - ... client.Client(openstack_client) - ... # use them - ... openstack_client.identity.tenants.list() - ... openstack_client.compute.servers.list() - """ - service_type = base_client_instance.service_type - if service_type and not hasattr(self, service_type): - setattr(self, service_type, base_client_instance) - - def authenticate(self): - self.auth_plugin.authenticate(self) - # Store the authentication results in the keyring for later requests - if self.keyring_saver: - self.keyring_saver.save(self) - - -class BaseClient(object): - """Top-level object to access the OpenStack API. - - This client uses :class:`HTTPClient` to send requests. :class:`HTTPClient` - will handle a bunch of issues such as authentication. - """ - - service_type = None - endpoint_type = None # "publicURL" will be used - cached_endpoint = None - - def __init__(self, http_client, extensions=None): - self.http_client = http_client - http_client.add_client(self) - - # Add in any extensions... - if extensions: - for extension in extensions: - if extension.manager_class: - setattr(self, extension.name, - extension.manager_class(self)) - - def client_request(self, method, url, **kwargs): - return self.http_client.client_request( - self, method, url, **kwargs) - - def head(self, url, **kwargs): - return self.client_request("HEAD", url, **kwargs) - - def get(self, url, **kwargs): - return self.client_request("GET", url, **kwargs) - - def post(self, url, **kwargs): - return self.client_request("POST", url, **kwargs) - - def put(self, url, **kwargs): - return self.client_request("PUT", url, **kwargs) - - def delete(self, url, **kwargs): - return self.client_request("DELETE", url, **kwargs) - - def patch(self, url, **kwargs): - return self.client_request("PATCH", url, **kwargs) - - @staticmethod - def get_class(api_name, version, version_map): - """Returns the client class for the requested API version - - :param api_name: the name of the API, e.g. 'compute', 'image', etc - :param version: the requested API version - :param version_map: a dict of client classes keyed by version - :rtype: a client class for the requested API version - """ - try: - client_path = version_map[str(version)] - except (KeyError, ValueError): - msg = _("Invalid %(api_name)s client version '%(version)s'. " - "Must be one of: %(version_map)s") % { - 'api_name': api_name, - 'version': version, - 'version_map': ', '.join(version_map.keys()) - } - raise exceptions.UnsupportedVersion(msg) - - return importutils.import_class(client_path) diff --git a/logaas/openstack/common/apiclient/exceptions.py b/logaas/openstack/common/apiclient/exceptions.py deleted file mode 100644 index 05cc4bb..0000000 --- a/logaas/openstack/common/apiclient/exceptions.py +++ /dev/null @@ -1,466 +0,0 @@ -# Copyright 2010 Jacob Kaplan-Moss -# Copyright 2011 Nebula, Inc. -# Copyright 2013 Alessio Ababilov -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Exception definitions. -""" - -import inspect -import sys - -import six - -from logaas.openstack.common.gettextutils import _ - - -class ClientException(Exception): - """The base exception class for all exceptions this library raises. - """ - pass - - -class MissingArgs(ClientException): - """Supplied arguments are not sufficient for calling a function.""" - def __init__(self, missing): - self.missing = missing - msg = _("Missing arguments: %s") % ", ".join(missing) - super(MissingArgs, self).__init__(msg) - - -class ValidationError(ClientException): - """Error in validation on API client side.""" - pass - - -class UnsupportedVersion(ClientException): - """User is trying to use an unsupported version of the API.""" - pass - - -class CommandError(ClientException): - """Error in CLI tool.""" - pass - - -class AuthorizationFailure(ClientException): - """Cannot authorize API client.""" - pass - - -class ConnectionRefused(ClientException): - """Cannot connect to API service.""" - pass - - -class AuthPluginOptionsMissing(AuthorizationFailure): - """Auth plugin misses some options.""" - def __init__(self, opt_names): - super(AuthPluginOptionsMissing, self).__init__( - _("Authentication failed. Missing options: %s") % - ", ".join(opt_names)) - self.opt_names = opt_names - - -class AuthSystemNotFound(AuthorizationFailure): - """User has specified a AuthSystem that is not installed.""" - def __init__(self, auth_system): - super(AuthSystemNotFound, self).__init__( - _("AuthSystemNotFound: %s") % repr(auth_system)) - self.auth_system = auth_system - - -class NoUniqueMatch(ClientException): - """Multiple entities found instead of one.""" - pass - - -class EndpointException(ClientException): - """Something is rotten in Service Catalog.""" - pass - - -class EndpointNotFound(EndpointException): - """Could not find requested endpoint in Service Catalog.""" - pass - - -class AmbiguousEndpoints(EndpointException): - """Found more than one matching endpoint in Service Catalog.""" - def __init__(self, endpoints=None): - super(AmbiguousEndpoints, self).__init__( - _("AmbiguousEndpoints: %s") % repr(endpoints)) - self.endpoints = endpoints - - -class HttpError(ClientException): - """The base exception class for all HTTP exceptions. - """ - http_status = 0 - message = _("HTTP Error") - - def __init__(self, message=None, details=None, - response=None, request_id=None, - url=None, method=None, http_status=None): - self.http_status = http_status or self.http_status - self.message = message or self.message - self.details = details - self.request_id = request_id - self.response = response - self.url = url - self.method = method - formatted_string = "%s (HTTP %s)" % (self.message, self.http_status) - if request_id: - formatted_string += " (Request-ID: %s)" % request_id - super(HttpError, self).__init__(formatted_string) - - -class HTTPRedirection(HttpError): - """HTTP Redirection.""" - message = _("HTTP Redirection") - - -class HTTPClientError(HttpError): - """Client-side HTTP error. - - Exception for cases in which the client seems to have erred. - """ - message = _("HTTP Client Error") - - -class HttpServerError(HttpError): - """Server-side HTTP error. - - Exception for cases in which the server is aware that it has - erred or is incapable of performing the request. - """ - message = _("HTTP Server Error") - - -class MultipleChoices(HTTPRedirection): - """HTTP 300 - Multiple Choices. - - Indicates multiple options for the resource that the client may follow. - """ - - http_status = 300 - message = _("Multiple Choices") - - -class BadRequest(HTTPClientError): - """HTTP 400 - Bad Request. - - The request cannot be fulfilled due to bad syntax. - """ - http_status = 400 - message = _("Bad Request") - - -class Unauthorized(HTTPClientError): - """HTTP 401 - Unauthorized. - - Similar to 403 Forbidden, but specifically for use when authentication - is required and has failed or has not yet been provided. - """ - http_status = 401 - message = _("Unauthorized") - - -class PaymentRequired(HTTPClientError): - """HTTP 402 - Payment Required. - - Reserved for future use. - """ - http_status = 402 - message = _("Payment Required") - - -class Forbidden(HTTPClientError): - """HTTP 403 - Forbidden. - - The request was a valid request, but the server is refusing to respond - to it. - """ - http_status = 403 - message = _("Forbidden") - - -class NotFound(HTTPClientError): - """HTTP 404 - Not Found. - - The requested resource could not be found but may be available again - in the future. - """ - http_status = 404 - message = _("Not Found") - - -class MethodNotAllowed(HTTPClientError): - """HTTP 405 - Method Not Allowed. - - A request was made of a resource using a request method not supported - by that resource. - """ - http_status = 405 - message = _("Method Not Allowed") - - -class NotAcceptable(HTTPClientError): - """HTTP 406 - Not Acceptable. - - The requested resource is only capable of generating content not - acceptable according to the Accept headers sent in the request. - """ - http_status = 406 - message = _("Not Acceptable") - - -class ProxyAuthenticationRequired(HTTPClientError): - """HTTP 407 - Proxy Authentication Required. - - The client must first authenticate itself with the proxy. - """ - http_status = 407 - message = _("Proxy Authentication Required") - - -class RequestTimeout(HTTPClientError): - """HTTP 408 - Request Timeout. - - The server timed out waiting for the request. - """ - http_status = 408 - message = _("Request Timeout") - - -class Conflict(HTTPClientError): - """HTTP 409 - Conflict. - - Indicates that the request could not be processed because of conflict - in the request, such as an edit conflict. - """ - http_status = 409 - message = _("Conflict") - - -class Gone(HTTPClientError): - """HTTP 410 - Gone. - - Indicates that the resource requested is no longer available and will - not be available again. - """ - http_status = 410 - message = _("Gone") - - -class LengthRequired(HTTPClientError): - """HTTP 411 - Length Required. - - The request did not specify the length of its content, which is - required by the requested resource. - """ - http_status = 411 - message = _("Length Required") - - -class PreconditionFailed(HTTPClientError): - """HTTP 412 - Precondition Failed. - - The server does not meet one of the preconditions that the requester - put on the request. - """ - http_status = 412 - message = _("Precondition Failed") - - -class RequestEntityTooLarge(HTTPClientError): - """HTTP 413 - Request Entity Too Large. - - The request is larger than the server is willing or able to process. - """ - http_status = 413 - message = _("Request Entity Too Large") - - def __init__(self, *args, **kwargs): - try: - self.retry_after = int(kwargs.pop('retry_after')) - except (KeyError, ValueError): - self.retry_after = 0 - - super(RequestEntityTooLarge, self).__init__(*args, **kwargs) - - -class RequestUriTooLong(HTTPClientError): - """HTTP 414 - Request-URI Too Long. - - The URI provided was too long for the server to process. - """ - http_status = 414 - message = _("Request-URI Too Long") - - -class UnsupportedMediaType(HTTPClientError): - """HTTP 415 - Unsupported Media Type. - - The request entity has a media type which the server or resource does - not support. - """ - http_status = 415 - message = _("Unsupported Media Type") - - -class RequestedRangeNotSatisfiable(HTTPClientError): - """HTTP 416 - Requested Range Not Satisfiable. - - The client has asked for a portion of the file, but the server cannot - supply that portion. - """ - http_status = 416 - message = _("Requested Range Not Satisfiable") - - -class ExpectationFailed(HTTPClientError): - """HTTP 417 - Expectation Failed. - - The server cannot meet the requirements of the Expect request-header field. - """ - http_status = 417 - message = _("Expectation Failed") - - -class UnprocessableEntity(HTTPClientError): - """HTTP 422 - Unprocessable Entity. - - The request was well-formed but was unable to be followed due to semantic - errors. - """ - http_status = 422 - message = _("Unprocessable Entity") - - -class InternalServerError(HttpServerError): - """HTTP 500 - Internal Server Error. - - A generic error message, given when no more specific message is suitable. - """ - http_status = 500 - message = _("Internal Server Error") - - -# NotImplemented is a python keyword. -class HttpNotImplemented(HttpServerError): - """HTTP 501 - Not Implemented. - - The server either does not recognize the request method, or it lacks - the ability to fulfill the request. - """ - http_status = 501 - message = _("Not Implemented") - - -class BadGateway(HttpServerError): - """HTTP 502 - Bad Gateway. - - The server was acting as a gateway or proxy and received an invalid - response from the upstream server. - """ - http_status = 502 - message = _("Bad Gateway") - - -class ServiceUnavailable(HttpServerError): - """HTTP 503 - Service Unavailable. - - The server is currently unavailable. - """ - http_status = 503 - message = _("Service Unavailable") - - -class GatewayTimeout(HttpServerError): - """HTTP 504 - Gateway Timeout. - - The server was acting as a gateway or proxy and did not receive a timely - response from the upstream server. - """ - http_status = 504 - message = _("Gateway Timeout") - - -class HttpVersionNotSupported(HttpServerError): - """HTTP 505 - HttpVersion Not Supported. - - The server does not support the HTTP protocol version used in the request. - """ - http_status = 505 - message = _("HTTP Version Not Supported") - - -# _code_map contains all the classes that have http_status attribute. -_code_map = dict( - (getattr(obj, 'http_status', None), obj) - for name, obj in six.iteritems(vars(sys.modules[__name__])) - if inspect.isclass(obj) and getattr(obj, 'http_status', False) -) - - -def from_response(response, method, url): - """Returns an instance of :class:`HttpError` or subclass based on response. - - :param response: instance of `requests.Response` class - :param method: HTTP method used for request - :param url: URL used for request - """ - - req_id = response.headers.get("x-openstack-request-id") - #NOTE(hdd) true for older versions of nova and cinder - if not req_id: - req_id = response.headers.get("x-compute-request-id") - kwargs = { - "http_status": response.status_code, - "response": response, - "method": method, - "url": url, - "request_id": req_id, - } - if "retry-after" in response.headers: - kwargs["retry_after"] = response.headers["retry-after"] - - content_type = response.headers.get("Content-Type", "") - if content_type.startswith("application/json"): - try: - body = response.json() - except ValueError: - pass - else: - if isinstance(body, dict): - error = list(body.values())[0] - kwargs["message"] = error.get("message") - kwargs["details"] = error.get("details") - elif content_type.startswith("text/"): - kwargs["details"] = response.text - - try: - cls = _code_map[response.status_code] - except KeyError: - if 500 <= response.status_code < 600: - cls = HttpServerError - elif 400 <= response.status_code < 500: - cls = HTTPClientError - else: - cls = HttpError - return cls(**kwargs) diff --git a/logaas/openstack/common/apiclient/fake_client.py b/logaas/openstack/common/apiclient/fake_client.py deleted file mode 100644 index 2bd4e72..0000000 --- a/logaas/openstack/common/apiclient/fake_client.py +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -A fake server that "responds" to API methods with pre-canned responses. - -All of these responses come from the spec, so if for some reason the spec's -wrong the tests might raise AssertionError. I've indicated in comments the -places where actual behavior differs from the spec. -""" - -# W0102: Dangerous default value %s as argument -# pylint: disable=W0102 - -import json - -import requests -import six -from six.moves.urllib import parse - -from logaas.openstack.common.apiclient import client - - -def assert_has_keys(dct, required=[], optional=[]): - for k in required: - try: - assert k in dct - except AssertionError: - extra_keys = set(dct.keys()).difference(set(required + optional)) - raise AssertionError("found unexpected keys: %s" % - list(extra_keys)) - - -class TestResponse(requests.Response): - """Wrap requests.Response and provide a convenient initialization. - """ - - def __init__(self, data): - super(TestResponse, self).__init__() - self._content_consumed = True - if isinstance(data, dict): - self.status_code = data.get('status_code', 200) - # Fake the text attribute to streamline Response creation - text = data.get('text', "") - if isinstance(text, (dict, list)): - self._content = json.dumps(text) - default_headers = { - "Content-Type": "application/json", - } - else: - self._content = text - default_headers = {} - if six.PY3 and isinstance(self._content, six.string_types): - self._content = self._content.encode('utf-8', 'strict') - self.headers = data.get('headers') or default_headers - else: - self.status_code = data - - def __eq__(self, other): - return (self.status_code == other.status_code and - self.headers == other.headers and - self._content == other._content) - - -class FakeHTTPClient(client.HTTPClient): - - def __init__(self, *args, **kwargs): - self.callstack = [] - self.fixtures = kwargs.pop("fixtures", None) or {} - if not args and not "auth_plugin" in kwargs: - args = (None, ) - super(FakeHTTPClient, self).__init__(*args, **kwargs) - - def assert_called(self, method, url, body=None, pos=-1): - """Assert than an API method was just called. - """ - expected = (method, url) - called = self.callstack[pos][0:2] - assert self.callstack, \ - "Expected %s %s but no calls were made." % expected - - assert expected == called, 'Expected %s %s; got %s %s' % \ - (expected + called) - - if body is not None: - if self.callstack[pos][3] != body: - raise AssertionError('%r != %r' % - (self.callstack[pos][3], body)) - - def assert_called_anytime(self, method, url, body=None): - """Assert than an API method was called anytime in the test. - """ - expected = (method, url) - - assert self.callstack, \ - "Expected %s %s but no calls were made." % expected - - found = False - entry = None - for entry in self.callstack: - if expected == entry[0:2]: - found = True - break - - assert found, 'Expected %s %s; got %s' % \ - (method, url, self.callstack) - if body is not None: - assert entry[3] == body, "%s != %s" % (entry[3], body) - - self.callstack = [] - - def clear_callstack(self): - self.callstack = [] - - def authenticate(self): - pass - - def client_request(self, client, method, url, **kwargs): - # Check that certain things are called correctly - if method in ["GET", "DELETE"]: - assert "json" not in kwargs - - # Note the call - self.callstack.append( - (method, - url, - kwargs.get("headers") or {}, - kwargs.get("json") or kwargs.get("data"))) - try: - fixture = self.fixtures[url][method] - except KeyError: - pass - else: - return TestResponse({"headers": fixture[0], - "text": fixture[1]}) - - # Call the method - args = parse.parse_qsl(parse.urlparse(url)[4]) - kwargs.update(args) - munged_url = url.rsplit('?', 1)[0] - munged_url = munged_url.strip('/').replace('/', '_').replace('.', '_') - munged_url = munged_url.replace('-', '_') - - callback = "%s_%s" % (method.lower(), munged_url) - - if not hasattr(self, callback): - raise AssertionError('Called unknown API method: %s %s, ' - 'expected fakes method name: %s' % - (method, url, callback)) - - resp = getattr(self, callback)(**kwargs) - if len(resp) == 3: - status, headers, body = resp - else: - status, body = resp - headers = {} - return TestResponse({ - "status_code": status, - "text": body, - "headers": headers, - }) diff --git a/logaas/openstack/common/cliutils.py b/logaas/openstack/common/cliutils.py deleted file mode 100644 index e978c78..0000000 --- a/logaas/openstack/common/cliutils.py +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# W0603: Using the global statement -# W0621: Redefining name %s from outer scope -# pylint: disable=W0603,W0621 - -from __future__ import print_function - -import getpass -import inspect -import os -import sys -import textwrap - -import prettytable -import six -from six import moves - -from logaas.openstack.common.apiclient import exceptions -from logaas.openstack.common.gettextutils import _ -from logaas.openstack.common import strutils -from logaas.openstack.common import uuidutils - - -def validate_args(fn, *args, **kwargs): - """Check that the supplied args are sufficient for calling a function. - - >>> validate_args(lambda a: None) - Traceback (most recent call last): - ... - MissingArgs: Missing argument(s): a - >>> validate_args(lambda a, b, c, d: None, 0, c=1) - Traceback (most recent call last): - ... - MissingArgs: Missing argument(s): b, d - - :param fn: the function to check - :param arg: the positional arguments supplied - :param kwargs: the keyword arguments supplied - """ - argspec = inspect.getargspec(fn) - - num_defaults = len(argspec.defaults or []) - required_args = argspec.args[:len(argspec.args) - num_defaults] - - def isbound(method): - return getattr(method, '__self__', None) is not None - - if isbound(fn): - required_args.pop(0) - - missing = [arg for arg in required_args if arg not in kwargs] - missing = missing[len(args):] - if missing: - raise exceptions.MissingArgs(missing) - - -def arg(*args, **kwargs): - """Decorator for CLI args. - - Example: - - >>> @arg("name", help="Name of the new entity") - ... def entity_create(args): - ... pass - """ - def _decorator(func): - add_arg(func, *args, **kwargs) - return func - return _decorator - - -def env(*args, **kwargs): - """Returns the first environment variable set. - - If all are empty, defaults to '' or keyword arg `default`. - """ - for arg in args: - value = os.environ.get(arg) - if value: - return value - return kwargs.get('default', '') - - -def add_arg(func, *args, **kwargs): - """Bind CLI arguments to a shell.py `do_foo` function.""" - - if not hasattr(func, 'arguments'): - func.arguments = [] - - # NOTE(sirp): avoid dups that can occur when the module is shared across - # tests. - if (args, kwargs) not in func.arguments: - # Because of the semantics of decorator composition if we just append - # to the options list positional options will appear to be backwards. - func.arguments.insert(0, (args, kwargs)) - - -def unauthenticated(func): - """Adds 'unauthenticated' attribute to decorated function. - - Usage: - - >>> @unauthenticated - ... def mymethod(f): - ... pass - """ - func.unauthenticated = True - return func - - -def isunauthenticated(func): - """Checks if the function does not require authentication. - - Mark such functions with the `@unauthenticated` decorator. - - :returns: bool - """ - return getattr(func, 'unauthenticated', False) - - -def print_list(objs, fields, formatters=None, sortby_index=0, - mixed_case_fields=None): - """Print a list or objects as a table, one row per object. - - :param objs: iterable of :class:`Resource` - :param fields: attributes that correspond to columns, in order - :param formatters: `dict` of callables for field formatting - :param sortby_index: index of the field for sorting table rows - :param mixed_case_fields: fields corresponding to object attributes that - have mixed case names (e.g., 'serverId') - """ - formatters = formatters or {} - mixed_case_fields = mixed_case_fields or [] - if sortby_index is None: - kwargs = {} - else: - kwargs = {'sortby': fields[sortby_index]} - pt = prettytable.PrettyTable(fields, caching=False) - pt.align = 'l' - - for o in objs: - row = [] - for field in fields: - if field in formatters: - row.append(formatters[field](o)) - else: - if field in mixed_case_fields: - field_name = field.replace(' ', '_') - else: - field_name = field.lower().replace(' ', '_') - data = getattr(o, field_name, '') - row.append(data) - pt.add_row(row) - - print(strutils.safe_encode(pt.get_string(**kwargs))) - - -def print_dict(dct, dict_property="Property", wrap=0): - """Print a `dict` as a table of two columns. - - :param dct: `dict` to print - :param dict_property: name of the first column - :param wrap: wrapping for the second column - """ - pt = prettytable.PrettyTable([dict_property, 'Value'], caching=False) - pt.align = 'l' - for k, v in six.iteritems(dct): - # convert dict to str to check length - if isinstance(v, dict): - v = six.text_type(v) - if wrap > 0: - v = textwrap.fill(six.text_type(v), wrap) - # if value has a newline, add in multiple rows - # e.g. fault with stacktrace - if v and isinstance(v, six.string_types) and r'\n' in v: - lines = v.strip().split(r'\n') - col1 = k - for line in lines: - pt.add_row([col1, line]) - col1 = '' - else: - pt.add_row([k, v]) - print(strutils.safe_encode(pt.get_string())) - - -def get_password(max_password_prompts=3): - """Read password from TTY.""" - verify = strutils.bool_from_string(env("OS_VERIFY_PASSWORD")) - pw = None - if hasattr(sys.stdin, "isatty") and sys.stdin.isatty(): - # Check for Ctrl-D - try: - for __ in moves.range(max_password_prompts): - pw1 = getpass.getpass("OS Password: ") - if verify: - pw2 = getpass.getpass("Please verify: ") - else: - pw2 = pw1 - if pw1 == pw2 and pw1: - pw = pw1 - break - except EOFError: - pass - return pw - - -def find_resource(manager, name_or_id, **find_args): - """Look for resource in a given manager. - - Used as a helper for the _find_* methods. - Example: - - def _find_hypervisor(cs, hypervisor): - #Get a hypervisor by name or ID. - return cliutils.find_resource(cs.hypervisors, hypervisor) - """ - # first try to get entity as integer id - try: - return manager.get(int(name_or_id)) - except (TypeError, ValueError, exceptions.NotFound): - pass - - # now try to get entity as uuid - try: - if six.PY2: - tmp_id = strutils.safe_encode(name_or_id) - else: - tmp_id = strutils.safe_decode(name_or_id) - - if uuidutils.is_uuid_like(tmp_id): - return manager.get(tmp_id) - except (TypeError, ValueError, exceptions.NotFound): - pass - - # for str id which is not uuid - if getattr(manager, 'is_alphanum_id_allowed', False): - try: - return manager.get(name_or_id) - except exceptions.NotFound: - pass - - try: - try: - return manager.find(human_id=name_or_id, **find_args) - except exceptions.NotFound: - pass - - # finally try to find entity by name - try: - resource = getattr(manager, 'resource_class', None) - name_attr = resource.NAME_ATTR if resource else 'name' - kwargs = {name_attr: name_or_id} - kwargs.update(find_args) - return manager.find(**kwargs) - except exceptions.NotFound: - msg = _("No %(name)s with a name or " - "ID of '%(name_or_id)s' exists.") % \ - { - "name": manager.resource_class.__name__.lower(), - "name_or_id": name_or_id - } - raise exceptions.CommandError(msg) - except exceptions.NoUniqueMatch: - msg = _("Multiple %(name)s matches found for " - "'%(name_or_id)s', use an ID to be more specific.") % \ - { - "name": manager.resource_class.__name__.lower(), - "name_or_id": name_or_id - } - raise exceptions.CommandError(msg) - - -def service_type(stype): - """Adds 'service_type' attribute to decorated function. - - Usage: - @service_type('volume') - def mymethod(f): - ... - """ - def inner(f): - f.service_type = stype - return f - return inner - - -def get_service_type(f): - """Retrieves service type from function.""" - return getattr(f, 'service_type', None) - - -def pretty_choice_list(l): - return ', '.join("'%s'" % i for i in l) - - -def exit(msg=''): - if msg: - print (msg, file=sys.stderr) - sys.exit(1) diff --git a/logaas/openstack/common/config/__init__.py b/logaas/openstack/common/config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/logaas/openstack/common/config/generator.py b/logaas/openstack/common/config/generator.py deleted file mode 100644 index 863e528..0000000 --- a/logaas/openstack/common/config/generator.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2012 SINA Corporation -# Copyright 2014 Cisco Systems, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -"""Extracts OpenStack config option info from module(s).""" - -from __future__ import print_function - -import argparse -import imp -import os -import re -import socket -import sys -import textwrap - -from oslo.config import cfg -import six -import stevedore.named - -from logaas.openstack.common import gettextutils -from logaas.openstack.common import importutils - -gettextutils.install('logaas') - -STROPT = "StrOpt" -BOOLOPT = "BoolOpt" -INTOPT = "IntOpt" -FLOATOPT = "FloatOpt" -LISTOPT = "ListOpt" -DICTOPT = "DictOpt" -MULTISTROPT = "MultiStrOpt" - -OPT_TYPES = { - STROPT: 'string value', - BOOLOPT: 'boolean value', - INTOPT: 'integer value', - FLOATOPT: 'floating point value', - LISTOPT: 'list value', - DICTOPT: 'dict value', - MULTISTROPT: 'multi valued', -} - -OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT, - FLOATOPT, LISTOPT, DICTOPT, - MULTISTROPT])) - -PY_EXT = ".py" -BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), - "../../../../")) -WORDWRAP_WIDTH = 60 - - -def raise_extension_exception(extmanager, ep, err): - raise - - -def generate(argv): - parser = argparse.ArgumentParser( - description='generate sample configuration file', - ) - parser.add_argument('-m', dest='modules', action='append') - parser.add_argument('-l', dest='libraries', action='append') - parser.add_argument('srcfiles', nargs='*') - parsed_args = parser.parse_args(argv) - - mods_by_pkg = dict() - for filepath in parsed_args.srcfiles: - pkg_name = filepath.split(os.sep)[1] - mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]), - os.path.basename(filepath).split('.')[0]]) - mods_by_pkg.setdefault(pkg_name, list()).append(mod_str) - # NOTE(lzyeval): place top level modules before packages - pkg_names = sorted(pkg for pkg in mods_by_pkg if pkg.endswith(PY_EXT)) - ext_names = sorted(pkg for pkg in mods_by_pkg if pkg not in pkg_names) - pkg_names.extend(ext_names) - - # opts_by_group is a mapping of group name to an options list - # The options list is a list of (module, options) tuples - opts_by_group = {'DEFAULT': []} - - if parsed_args.modules: - for module_name in parsed_args.modules: - module = _import_module(module_name) - if module: - for group, opts in _list_opts(module): - opts_by_group.setdefault(group, []).append((module_name, - opts)) - - # Look for entry points defined in libraries (or applications) for - # option discovery, and include their return values in the output. - # - # Each entry point should be a function returning an iterable - # of pairs with the group name (or None for the default group) - # and the list of Opt instances for that group. - if parsed_args.libraries: - loader = stevedore.named.NamedExtensionManager( - 'oslo.config.opts', - names=list(set(parsed_args.libraries)), - invoke_on_load=False, - on_load_failure_callback=raise_extension_exception - ) - for ext in loader: - for group, opts in ext.plugin(): - opt_list = opts_by_group.setdefault(group or 'DEFAULT', []) - opt_list.append((ext.name, opts)) - - for pkg_name in pkg_names: - mods = mods_by_pkg.get(pkg_name) - mods.sort() - for mod_str in mods: - if mod_str.endswith('.__init__'): - mod_str = mod_str[:mod_str.rfind(".")] - - mod_obj = _import_module(mod_str) - if not mod_obj: - raise RuntimeError("Unable to import module %s" % mod_str) - - for group, opts in _list_opts(mod_obj): - opts_by_group.setdefault(group, []).append((mod_str, opts)) - - print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', [])) - for group in sorted(opts_by_group.keys()): - print_group_opts(group, opts_by_group[group]) - - -def _import_module(mod_str): - try: - if mod_str.startswith('bin.'): - imp.load_source(mod_str[4:], os.path.join('bin', mod_str[4:])) - return sys.modules[mod_str[4:]] - else: - return importutils.import_module(mod_str) - except Exception as e: - sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e))) - return None - - -def _is_in_group(opt, group): - "Check if opt is in group." - for value in group._opts.values(): - # NOTE(llu): Temporary workaround for bug #1262148, wait until - # newly released oslo.config support '==' operator. - if not(value['opt'] != opt): - return True - return False - - -def _guess_groups(opt, mod_obj): - # is it in the DEFAULT group? - if _is_in_group(opt, cfg.CONF): - return 'DEFAULT' - - # what other groups is it in? - for value in cfg.CONF.values(): - if isinstance(value, cfg.CONF.GroupAttr): - if _is_in_group(opt, value._group): - return value._group.name - - raise RuntimeError( - "Unable to find group for option %s, " - "maybe it's defined twice in the same group?" - % opt.name - ) - - -def _list_opts(obj): - def is_opt(o): - return (isinstance(o, cfg.Opt) and - not isinstance(o, cfg.SubCommandOpt)) - - opts = list() - for attr_str in dir(obj): - attr_obj = getattr(obj, attr_str) - if is_opt(attr_obj): - opts.append(attr_obj) - elif (isinstance(attr_obj, list) and - all(map(lambda x: is_opt(x), attr_obj))): - opts.extend(attr_obj) - - ret = {} - for opt in opts: - ret.setdefault(_guess_groups(opt, obj), []).append(opt) - return ret.items() - - -def print_group_opts(group, opts_by_module): - print("[%s]" % group) - print('') - for mod, opts in opts_by_module: - print('#') - print('# Options defined in %s' % mod) - print('#') - print('') - for opt in opts: - _print_opt(opt) - print('') - - -def _get_my_ip(): - try: - csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - csock.connect(('8.8.8.8', 80)) - (addr, port) = csock.getsockname() - csock.close() - return addr - except socket.error: - return None - - -def _sanitize_default(name, value): - """Set up a reasonably sensible default for pybasedir, my_ip and host.""" - hostname = socket.gethostname() - fqdn = socket.getfqdn() - if value.startswith(sys.prefix): - # NOTE(jd) Don't use os.path.join, because it is likely to think the - # second part is an absolute pathname and therefore drop the first - # part. - value = os.path.normpath("/usr/" + value[len(sys.prefix):]) - elif value.startswith(BASEDIR): - return value.replace(BASEDIR, '/usr/lib/python/site-packages') - elif BASEDIR in value: - return value.replace(BASEDIR, '') - elif value == _get_my_ip(): - return '10.0.0.1' - elif value in (hostname, fqdn): - if 'host' in name: - return 'logaas' - elif value.endswith(hostname): - return value.replace(hostname, 'logaas') - elif value.endswith(fqdn): - return value.replace(fqdn, 'logaas') - elif value.strip() != value: - return '"%s"' % value - return value - - -def _print_opt(opt): - opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help - if not opt_help: - sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name) - opt_help = "" - opt_type = None - try: - opt_type = OPTION_REGEX.search(str(type(opt))).group(0) - except (ValueError, AttributeError) as err: - sys.stderr.write("%s\n" % str(err)) - sys.exit(1) - opt_help = u'%s (%s)' % (opt_help, - OPT_TYPES[opt_type]) - print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))) - if opt.deprecated_opts: - for deprecated_opt in opt.deprecated_opts: - if deprecated_opt.name: - deprecated_group = (deprecated_opt.group if - deprecated_opt.group else "DEFAULT") - print('# Deprecated group/name - [%s]/%s' % - (deprecated_group, - deprecated_opt.name)) - try: - if opt_default is None: - print('#%s=' % opt_name) - elif opt_type == STROPT: - assert(isinstance(opt_default, six.string_types)) - print('#%s=%s' % (opt_name, _sanitize_default(opt_name, - opt_default))) - elif opt_type == BOOLOPT: - assert(isinstance(opt_default, bool)) - print('#%s=%s' % (opt_name, str(opt_default).lower())) - elif opt_type == INTOPT: - assert(isinstance(opt_default, int) and - not isinstance(opt_default, bool)) - print('#%s=%s' % (opt_name, opt_default)) - elif opt_type == FLOATOPT: - assert(isinstance(opt_default, float)) - print('#%s=%s' % (opt_name, opt_default)) - elif opt_type == LISTOPT: - assert(isinstance(opt_default, list)) - print('#%s=%s' % (opt_name, ','.join(opt_default))) - elif opt_type == DICTOPT: - assert(isinstance(opt_default, dict)) - opt_default_strlist = [str(key) + ':' + str(value) - for (key, value) in opt_default.items()] - print('#%s=%s' % (opt_name, ','.join(opt_default_strlist))) - elif opt_type == MULTISTROPT: - assert(isinstance(opt_default, list)) - if not opt_default: - opt_default = [''] - for default in opt_default: - print('#%s=%s' % (opt_name, default)) - print('') - except Exception: - sys.stderr.write('Error in option "%s"\n' % opt_name) - sys.exit(1) - - -def main(): - generate(sys.argv[1:]) - -if __name__ == '__main__': - main() diff --git a/logaas/openstack/common/context.py b/logaas/openstack/common/context.py deleted file mode 100644 index 3eeb445..0000000 --- a/logaas/openstack/common/context.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Simple class that stores security context information in the web request. - -Projects should subclass this class if they wish to enhance the request -context or provide additional information in their specific WSGI pipeline. -""" - -import itertools -import uuid - - -def generate_request_id(): - return b'req-' + str(uuid.uuid4()).encode('ascii') - - -class RequestContext(object): - - """Helper class to represent useful information about a request context. - - Stores information about the security context under which the user - accesses the system, as well as additional request information. - """ - - user_idt_format = '{user} {tenant} {domain} {user_domain} {p_domain}' - - def __init__(self, auth_token=None, user=None, tenant=None, domain=None, - user_domain=None, project_domain=None, is_admin=False, - read_only=False, show_deleted=False, request_id=None, - instance_uuid=None): - self.auth_token = auth_token - self.user = user - self.tenant = tenant - self.domain = domain - self.user_domain = user_domain - self.project_domain = project_domain - self.is_admin = is_admin - self.read_only = read_only - self.show_deleted = show_deleted - self.instance_uuid = instance_uuid - if not request_id: - request_id = generate_request_id() - self.request_id = request_id - - def to_dict(self): - user_idt = ( - self.user_idt_format.format(user=self.user or '-', - tenant=self.tenant or '-', - domain=self.domain or '-', - user_domain=self.user_domain or '-', - p_domain=self.project_domain or '-')) - - return {'user': self.user, - 'tenant': self.tenant, - 'domain': self.domain, - 'user_domain': self.user_domain, - 'project_domain': self.project_domain, - 'is_admin': self.is_admin, - 'read_only': self.read_only, - 'show_deleted': self.show_deleted, - 'auth_token': self.auth_token, - 'request_id': self.request_id, - 'instance_uuid': self.instance_uuid, - 'user_identity': user_idt} - - -def get_admin_context(show_deleted=False): - context = RequestContext(None, - tenant=None, - is_admin=True, - show_deleted=show_deleted) - return context - - -def get_context_from_function_and_args(function, args, kwargs): - """Find an arg of type RequestContext and return it. - - This is useful in a couple of decorators where we don't - know much about the function we're wrapping. - """ - - for arg in itertools.chain(kwargs.values(), args): - if isinstance(arg, RequestContext): - return arg - - return None - - -def is_user_context(context): - """Indicates if the request context is a normal user.""" - if not context: - return False - if context.is_admin: - return False - if not context.user_id or not context.project_id: - return False - return True diff --git a/logaas/openstack/common/db/__init__.py b/logaas/openstack/common/db/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/logaas/openstack/common/db/api.py b/logaas/openstack/common/db/api.py deleted file mode 100644 index 377fb24..0000000 --- a/logaas/openstack/common/db/api.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (c) 2013 Rackspace Hosting -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Multiple DB API backend support. - -A DB backend module should implement a method named 'get_backend' which -takes no arguments. The method can return any object that implements DB -API methods. -""" - -import functools -import logging -import threading -import time - -from logaas.openstack.common.db import exception -from logaas.openstack.common.gettextutils import _LE -from logaas.openstack.common import importutils - - -LOG = logging.getLogger(__name__) - - -def safe_for_db_retry(f): - """Enable db-retry for decorated function, if config option enabled.""" - f.__dict__['enable_retry'] = True - return f - - -class wrap_db_retry(object): - """Retry db.api methods, if DBConnectionError() raised - - Retry decorated db.api methods. If we enabled `use_db_reconnect` - in config, this decorator will be applied to all db.api functions, - marked with @safe_for_db_retry decorator. - Decorator catchs DBConnectionError() and retries function in a - loop until it succeeds, or until maximum retries count will be reached. - """ - - def __init__(self, retry_interval, max_retries, inc_retry_interval, - max_retry_interval): - super(wrap_db_retry, self).__init__() - - self.retry_interval = retry_interval - self.max_retries = max_retries - self.inc_retry_interval = inc_retry_interval - self.max_retry_interval = max_retry_interval - - def __call__(self, f): - @functools.wraps(f) - def wrapper(*args, **kwargs): - next_interval = self.retry_interval - remaining = self.max_retries - - while True: - try: - return f(*args, **kwargs) - except exception.DBConnectionError as e: - if remaining == 0: - LOG.exception(_LE('DB exceeded retry limit.')) - raise exception.DBError(e) - if remaining != -1: - remaining -= 1 - LOG.exception(_LE('DB connection error.')) - # NOTE(vsergeyev): We are using patched time module, so - # this effectively yields the execution - # context to another green thread. - time.sleep(next_interval) - if self.inc_retry_interval: - next_interval = min( - next_interval * 2, - self.max_retry_interval - ) - return wrapper - - -class DBAPI(object): - def __init__(self, backend_name, backend_mapping=None, lazy=False, - **kwargs): - """Initialize the chosen DB API backend. - - :param backend_name: name of the backend to load - :type backend_name: str - - :param backend_mapping: backend name -> module/class to load mapping - :type backend_mapping: dict - - :param lazy: load the DB backend lazily on the first DB API method call - :type lazy: bool - - Keyword arguments: - - :keyword use_db_reconnect: retry DB transactions on disconnect or not - :type use_db_reconnect: bool - - :keyword retry_interval: seconds between transaction retries - :type retry_interval: int - - :keyword inc_retry_interval: increase retry interval or not - :type inc_retry_interval: bool - - :keyword max_retry_interval: max interval value between retries - :type max_retry_interval: int - - :keyword max_retries: max number of retries before an error is raised - :type max_retries: int - - """ - - self._backend = None - self._backend_name = backend_name - self._backend_mapping = backend_mapping or {} - self._lock = threading.Lock() - - if not lazy: - self._load_backend() - - self.use_db_reconnect = kwargs.get('use_db_reconnect', False) - self.retry_interval = kwargs.get('retry_interval', 1) - self.inc_retry_interval = kwargs.get('inc_retry_interval', True) - self.max_retry_interval = kwargs.get('max_retry_interval', 10) - self.max_retries = kwargs.get('max_retries', 20) - - def _load_backend(self): - with self._lock: - if not self._backend: - # Import the untranslated name if we don't have a mapping - backend_path = self._backend_mapping.get(self._backend_name, - self._backend_name) - backend_mod = importutils.import_module(backend_path) - self._backend = backend_mod.get_backend() - - def __getattr__(self, key): - if not self._backend: - self._load_backend() - - attr = getattr(self._backend, key) - if not hasattr(attr, '__call__'): - return attr - # NOTE(vsergeyev): If `use_db_reconnect` option is set to True, retry - # DB API methods, decorated with @safe_for_db_retry - # on disconnect. - if self.use_db_reconnect and hasattr(attr, 'enable_retry'): - attr = wrap_db_retry( - retry_interval=self.retry_interval, - max_retries=self.max_retries, - inc_retry_interval=self.inc_retry_interval, - max_retry_interval=self.max_retry_interval)(attr) - - return attr diff --git a/logaas/openstack/common/db/exception.py b/logaas/openstack/common/db/exception.py deleted file mode 100644 index 620c054..0000000 --- a/logaas/openstack/common/db/exception.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""DB related custom exceptions.""" - -import six - -from logaas.openstack.common.gettextutils import _ - - -class DBError(Exception): - """Wraps an implementation specific exception.""" - def __init__(self, inner_exception=None): - self.inner_exception = inner_exception - super(DBError, self).__init__(six.text_type(inner_exception)) - - -class DBDuplicateEntry(DBError): - """Wraps an implementation specific exception.""" - def __init__(self, columns=[], inner_exception=None): - self.columns = columns - super(DBDuplicateEntry, self).__init__(inner_exception) - - -class DBDeadlock(DBError): - def __init__(self, inner_exception=None): - super(DBDeadlock, self).__init__(inner_exception) - - -class DBInvalidUnicodeParameter(Exception): - message = _("Invalid Parameter: " - "Unicode is not supported by the current database.") - - -class DbMigrationError(DBError): - """Wraps migration specific exception.""" - def __init__(self, message=None): - super(DbMigrationError, self).__init__(message) - - -class DBConnectionError(DBError): - """Wraps connection specific exception.""" - pass diff --git a/logaas/openstack/common/db/options.py b/logaas/openstack/common/db/options.py deleted file mode 100644 index 6270c18..0000000 --- a/logaas/openstack/common/db/options.py +++ /dev/null @@ -1,171 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy - -from oslo.config import cfg - - -database_opts = [ - cfg.StrOpt('sqlite_db', - deprecated_group='DEFAULT', - default='logaas.sqlite', - help='The file name to use with SQLite'), - cfg.BoolOpt('sqlite_synchronous', - deprecated_group='DEFAULT', - default=True, - help='If True, SQLite uses synchronous mode'), - cfg.StrOpt('backend', - default='sqlalchemy', - deprecated_name='db_backend', - deprecated_group='DEFAULT', - help='The backend to use for db'), - cfg.StrOpt('connection', - help='The SQLAlchemy connection string used to connect to the ' - 'database', - secret=True, - deprecated_opts=[cfg.DeprecatedOpt('sql_connection', - group='DEFAULT'), - cfg.DeprecatedOpt('sql_connection', - group='DATABASE'), - cfg.DeprecatedOpt('connection', - group='sql'), ]), - cfg.StrOpt('mysql_sql_mode', - default='TRADITIONAL', - help='The SQL mode to be used for MySQL sessions. ' - 'This option, including the default, overrides any ' - 'server-set SQL mode. To use whatever SQL mode ' - 'is set by the server configuration, ' - 'set this to no value. Example: mysql_sql_mode='), - cfg.IntOpt('idle_timeout', - default=3600, - deprecated_opts=[cfg.DeprecatedOpt('sql_idle_timeout', - group='DEFAULT'), - cfg.DeprecatedOpt('sql_idle_timeout', - group='DATABASE'), - cfg.DeprecatedOpt('idle_timeout', - group='sql')], - help='Timeout before idle sql connections are reaped'), - cfg.IntOpt('min_pool_size', - default=1, - deprecated_opts=[cfg.DeprecatedOpt('sql_min_pool_size', - group='DEFAULT'), - cfg.DeprecatedOpt('sql_min_pool_size', - group='DATABASE')], - help='Minimum number of SQL connections to keep open in a ' - 'pool'), - cfg.IntOpt('max_pool_size', - default=None, - deprecated_opts=[cfg.DeprecatedOpt('sql_max_pool_size', - group='DEFAULT'), - cfg.DeprecatedOpt('sql_max_pool_size', - group='DATABASE')], - help='Maximum number of SQL connections to keep open in a ' - 'pool'), - cfg.IntOpt('max_retries', - default=10, - deprecated_opts=[cfg.DeprecatedOpt('sql_max_retries', - group='DEFAULT'), - cfg.DeprecatedOpt('sql_max_retries', - group='DATABASE')], - help='Maximum db connection retries during startup. ' - '(setting -1 implies an infinite retry count)'), - cfg.IntOpt('retry_interval', - default=10, - deprecated_opts=[cfg.DeprecatedOpt('sql_retry_interval', - group='DEFAULT'), - cfg.DeprecatedOpt('reconnect_interval', - group='DATABASE')], - help='Interval between retries of opening a sql connection'), - cfg.IntOpt('max_overflow', - default=None, - deprecated_opts=[cfg.DeprecatedOpt('sql_max_overflow', - group='DEFAULT'), - cfg.DeprecatedOpt('sqlalchemy_max_overflow', - group='DATABASE')], - help='If set, use this value for max_overflow with sqlalchemy'), - cfg.IntOpt('connection_debug', - default=0, - deprecated_opts=[cfg.DeprecatedOpt('sql_connection_debug', - group='DEFAULT')], - help='Verbosity of SQL debugging information. 0=None, ' - '100=Everything'), - cfg.BoolOpt('connection_trace', - default=False, - deprecated_opts=[cfg.DeprecatedOpt('sql_connection_trace', - group='DEFAULT')], - help='Add python stack traces to SQL as comment strings'), - cfg.IntOpt('pool_timeout', - default=None, - deprecated_opts=[cfg.DeprecatedOpt('sqlalchemy_pool_timeout', - group='DATABASE')], - help='If set, use this value for pool_timeout with sqlalchemy'), - cfg.BoolOpt('use_db_reconnect', - default=False, - help='Enable the experimental use of database reconnect ' - 'on connection lost'), - cfg.IntOpt('db_retry_interval', - default=1, - help='seconds between db connection retries'), - cfg.BoolOpt('db_inc_retry_interval', - default=True, - help='Whether to increase interval between db connection ' - 'retries, up to db_max_retry_interval'), - cfg.IntOpt('db_max_retry_interval', - default=10, - help='max seconds between db connection retries, if ' - 'db_inc_retry_interval is enabled'), - cfg.IntOpt('db_max_retries', - default=20, - help='maximum db connection retries before error is raised. ' - '(setting -1 implies an infinite retry count)'), -] - -CONF = cfg.CONF -CONF.register_opts(database_opts, 'database') - - -def set_defaults(sql_connection, sqlite_db, max_pool_size=None, - max_overflow=None, pool_timeout=None): - """Set defaults for configuration variables.""" - cfg.set_defaults(database_opts, - connection=sql_connection, - sqlite_db=sqlite_db) - # Update the QueuePool defaults - if max_pool_size is not None: - cfg.set_defaults(database_opts, - max_pool_size=max_pool_size) - if max_overflow is not None: - cfg.set_defaults(database_opts, - max_overflow=max_overflow) - if pool_timeout is not None: - cfg.set_defaults(database_opts, - pool_timeout=pool_timeout) - - -def list_opts(): - """Returns a list of oslo.config options available in the library. - - The returned list includes all oslo.config options which may be registered - at runtime by the library. - - Each element of the list is a tuple. The first element is the name of the - group under which the list of elements in the second element will be - registered. A group name of None corresponds to the [DEFAULT] group in - config files. - - The purpose of this is to allow tools like the Oslo sample config file - generator to discover the options exposed to users by this library. - - :returns: a list of (group_name, opts) tuples - """ - return [('database', copy.deepcopy(database_opts))] diff --git a/logaas/openstack/common/db/sqlalchemy/__init__.py b/logaas/openstack/common/db/sqlalchemy/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/logaas/openstack/common/db/sqlalchemy/migration.py b/logaas/openstack/common/db/sqlalchemy/migration.py deleted file mode 100644 index 93e15a4..0000000 --- a/logaas/openstack/common/db/sqlalchemy/migration.py +++ /dev/null @@ -1,278 +0,0 @@ -# coding: utf-8 -# -# Copyright (c) 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# Base on code in migrate/changeset/databases/sqlite.py which is under -# the following license: -# -# The MIT License -# -# Copyright (c) 2009 Evan Rosson, Jan Dittberner, Domen Kožar -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -import os -import re - -from migrate.changeset import ansisql -from migrate.changeset.databases import sqlite -from migrate import exceptions as versioning_exceptions -from migrate.versioning import api as versioning_api -from migrate.versioning.repository import Repository -import sqlalchemy -from sqlalchemy.schema import UniqueConstraint - -from logaas.openstack.common.db import exception -from logaas.openstack.common.gettextutils import _ - - -def _get_unique_constraints(self, table): - """Retrieve information about existing unique constraints of the table - - This feature is needed for _recreate_table() to work properly. - Unfortunately, it's not available in sqlalchemy 0.7.x/0.8.x. - - """ - - data = table.metadata.bind.execute( - """SELECT sql - FROM sqlite_master - WHERE - type='table' AND - name=:table_name""", - table_name=table.name - ).fetchone()[0] - - UNIQUE_PATTERN = "CONSTRAINT (\w+) UNIQUE \(([^\)]+)\)" - return [ - UniqueConstraint( - *[getattr(table.columns, c.strip(' "')) for c in cols.split(",")], - name=name - ) - for name, cols in re.findall(UNIQUE_PATTERN, data) - ] - - -def _recreate_table(self, table, column=None, delta=None, omit_uniques=None): - """Recreate the table properly - - Unlike the corresponding original method of sqlalchemy-migrate this one - doesn't drop existing unique constraints when creating a new one. - - """ - - table_name = self.preparer.format_table(table) - - # we remove all indexes so as not to have - # problems during copy and re-create - for index in table.indexes: - index.drop() - - # reflect existing unique constraints - for uc in self._get_unique_constraints(table): - table.append_constraint(uc) - # omit given unique constraints when creating a new table if required - table.constraints = set([ - cons for cons in table.constraints - if omit_uniques is None or cons.name not in omit_uniques - ]) - - self.append('ALTER TABLE %s RENAME TO migration_tmp' % table_name) - self.execute() - - insertion_string = self._modify_table(table, column, delta) - - table.create(bind=self.connection) - self.append(insertion_string % {'table_name': table_name}) - self.execute() - self.append('DROP TABLE migration_tmp') - self.execute() - - -def _visit_migrate_unique_constraint(self, *p, **k): - """Drop the given unique constraint - - The corresponding original method of sqlalchemy-migrate just - raises NotImplemented error - - """ - - self.recreate_table(p[0].table, omit_uniques=[p[0].name]) - - -def patch_migrate(): - """A workaround for SQLite's inability to alter things - - SQLite abilities to alter tables are very limited (please read - http://www.sqlite.org/lang_altertable.html for more details). - E. g. one can't drop a column or a constraint in SQLite. The - workaround for this is to recreate the original table omitting - the corresponding constraint (or column). - - sqlalchemy-migrate library has recreate_table() method that - implements this workaround, but it does it wrong: - - - information about unique constraints of a table - is not retrieved. So if you have a table with one - unique constraint and a migration adding another one - you will end up with a table that has only the - latter unique constraint, and the former will be lost - - - dropping of unique constraints is not supported at all - - The proper way to fix this is to provide a pull-request to - sqlalchemy-migrate, but the project seems to be dead. So we - can go on with monkey-patching of the lib at least for now. - - """ - - # this patch is needed to ensure that recreate_table() doesn't drop - # existing unique constraints of the table when creating a new one - helper_cls = sqlite.SQLiteHelper - helper_cls.recreate_table = _recreate_table - helper_cls._get_unique_constraints = _get_unique_constraints - - # this patch is needed to be able to drop existing unique constraints - constraint_cls = sqlite.SQLiteConstraintDropper - constraint_cls.visit_migrate_unique_constraint = \ - _visit_migrate_unique_constraint - constraint_cls.__bases__ = (ansisql.ANSIColumnDropper, - sqlite.SQLiteConstraintGenerator) - - -def db_sync(engine, abs_path, version=None, init_version=0, sanity_check=True): - """Upgrade or downgrade a database. - - Function runs the upgrade() or downgrade() functions in change scripts. - - :param engine: SQLAlchemy engine instance for a given database - :param abs_path: Absolute path to migrate repository. - :param version: Database will upgrade/downgrade until this version. - If None - database will update to the latest - available version. - :param init_version: Initial database version - :param sanity_check: Require schema sanity checking for all tables - """ - - if version is not None: - try: - version = int(version) - except ValueError: - raise exception.DbMigrationError( - message=_("version should be an integer")) - - current_version = db_version(engine, abs_path, init_version) - repository = _find_migrate_repo(abs_path) - if sanity_check: - _db_schema_sanity_check(engine) - if version is None or version > current_version: - return versioning_api.upgrade(engine, repository, version) - else: - return versioning_api.downgrade(engine, repository, - version) - - -def _db_schema_sanity_check(engine): - """Ensure all database tables were created with required parameters. - - :param engine: SQLAlchemy engine instance for a given database - - """ - - if engine.name == 'mysql': - onlyutf8_sql = ('SELECT TABLE_NAME,TABLE_COLLATION ' - 'from information_schema.TABLES ' - 'where TABLE_SCHEMA=%s and ' - 'TABLE_COLLATION NOT LIKE "%%utf8%%"') - - # NOTE(morganfainberg): exclude the sqlalchemy-migrate and alembic - # versioning tables from the tables we need to verify utf8 status on. - # Non-standard table names are not supported. - EXCLUDED_TABLES = ['migrate_version', 'alembic_version'] - - table_names = [res[0] for res in - engine.execute(onlyutf8_sql, engine.url.database) if - res[0].lower() not in EXCLUDED_TABLES] - - if len(table_names) > 0: - raise ValueError(_('Tables "%s" have non utf8 collation, ' - 'please make sure all tables are CHARSET=utf8' - ) % ','.join(table_names)) - - -def db_version(engine, abs_path, init_version): - """Show the current version of the repository. - - :param engine: SQLAlchemy engine instance for a given database - :param abs_path: Absolute path to migrate repository - :param version: Initial database version - """ - repository = _find_migrate_repo(abs_path) - try: - return versioning_api.db_version(engine, repository) - except versioning_exceptions.DatabaseNotControlledError: - meta = sqlalchemy.MetaData() - meta.reflect(bind=engine) - tables = meta.tables - if len(tables) == 0 or 'alembic_version' in tables: - db_version_control(engine, abs_path, version=init_version) - return versioning_api.db_version(engine, repository) - else: - raise exception.DbMigrationError( - message=_( - "The database is not under version control, but has " - "tables. Please stamp the current version of the schema " - "manually.")) - - -def db_version_control(engine, abs_path, version=None): - """Mark a database as under this repository's version control. - - Once a database is under version control, schema changes should - only be done via change scripts in this repository. - - :param engine: SQLAlchemy engine instance for a given database - :param abs_path: Absolute path to migrate repository - :param version: Initial database version - """ - repository = _find_migrate_repo(abs_path) - versioning_api.version_control(engine, repository, version) - return version - - -def _find_migrate_repo(abs_path): - """Get the project's change script repository - - :param abs_path: Absolute path to migrate repository - """ - if not os.path.exists(abs_path): - raise exception.DbMigrationError("Path %s not found" % abs_path) - return Repository(abs_path) diff --git a/logaas/openstack/common/db/sqlalchemy/models.py b/logaas/openstack/common/db/sqlalchemy/models.py deleted file mode 100644 index 802389c..0000000 --- a/logaas/openstack/common/db/sqlalchemy/models.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) 2011 X.commerce, a business unit of eBay Inc. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Piston Cloud Computing, Inc. -# Copyright 2012 Cloudscaling Group, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -SQLAlchemy models. -""" - -import six - -from sqlalchemy import Column, Integer -from sqlalchemy import DateTime -from sqlalchemy.orm import object_mapper - -from logaas.openstack.common import timeutils - - -class ModelBase(six.Iterator): - """Base class for models.""" - __table_initialized__ = False - - def save(self, session): - """Save this object.""" - - # NOTE(boris-42): This part of code should be look like: - # session.add(self) - # session.flush() - # But there is a bug in sqlalchemy and eventlet that - # raises NoneType exception if there is no running - # transaction and rollback is called. As long as - # sqlalchemy has this bug we have to create transaction - # explicitly. - with session.begin(subtransactions=True): - session.add(self) - session.flush() - - def __setitem__(self, key, value): - setattr(self, key, value) - - def __getitem__(self, key): - return getattr(self, key) - - def get(self, key, default=None): - return getattr(self, key, default) - - @property - def _extra_keys(self): - """Specifies custom fields - - Subclasses can override this property to return a list - of custom fields that should be included in their dict - representation. - - For reference check tests/db/sqlalchemy/test_models.py - """ - return [] - - def __iter__(self): - columns = list(dict(object_mapper(self).columns).keys()) - # NOTE(russellb): Allow models to specify other keys that can be looked - # up, beyond the actual db columns. An example would be the 'name' - # property for an Instance. - columns.extend(self._extra_keys) - self._i = iter(columns) - return self - - # In Python 3, __next__() has replaced next(). - def __next__(self): - n = six.advance_iterator(self._i) - return n, getattr(self, n) - - def next(self): - return self.__next__() - - def update(self, values): - """Make the model object behave like a dict.""" - for k, v in six.iteritems(values): - setattr(self, k, v) - - def iteritems(self): - """Make the model object behave like a dict. - - Includes attributes from joins. - """ - local = dict(self) - joined = dict([(k, v) for k, v in six.iteritems(self.__dict__) - if not k[0] == '_']) - local.update(joined) - return six.iteritems(local) - - -class TimestampMixin(object): - created_at = Column(DateTime, default=lambda: timeutils.utcnow()) - updated_at = Column(DateTime, onupdate=lambda: timeutils.utcnow()) - - -class SoftDeleteMixin(object): - deleted_at = Column(DateTime) - deleted = Column(Integer, default=0) - - def soft_delete(self, session): - """Mark this object as deleted.""" - self.deleted = self.id - self.deleted_at = timeutils.utcnow() - self.save(session=session) diff --git a/logaas/openstack/common/db/sqlalchemy/provision.py b/logaas/openstack/common/db/sqlalchemy/provision.py deleted file mode 100644 index 834650b..0000000 --- a/logaas/openstack/common/db/sqlalchemy/provision.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2013 Mirantis.inc -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Provision test environment for specific DB backends""" - -import argparse -import logging -import os -import random -import string - -from six import moves -import sqlalchemy - -from logaas.openstack.common.db import exception as exc - - -LOG = logging.getLogger(__name__) - - -def get_engine(uri): - """Engine creation - - Call the function without arguments to get admin connection. Admin - connection required to create temporary user and database for each - particular test. Otherwise use existing connection to recreate connection - to the temporary database. - """ - return sqlalchemy.create_engine(uri, poolclass=sqlalchemy.pool.NullPool) - - -def _execute_sql(engine, sql, driver): - """Initialize connection, execute sql query and close it.""" - try: - with engine.connect() as conn: - if driver == 'postgresql': - conn.connection.set_isolation_level(0) - for s in sql: - conn.execute(s) - except sqlalchemy.exc.OperationalError: - msg = ('%s does not match database admin ' - 'credentials or database does not exist.') - LOG.exception(msg % engine.url) - raise exc.DBConnectionError(msg % engine.url) - - -def create_database(engine): - """Provide temporary user and database for each particular test.""" - driver = engine.name - - auth = { - 'database': ''.join(random.choice(string.ascii_lowercase) - for i in moves.range(10)), - 'user': engine.url.username, - 'passwd': engine.url.password, - } - - sqls = [ - "drop database if exists %(database)s;", - "create database %(database)s;" - ] - - if driver == 'sqlite': - return 'sqlite:////tmp/%s' % auth['database'] - elif driver in ['mysql', 'postgresql']: - sql_query = map(lambda x: x % auth, sqls) - _execute_sql(engine, sql_query, driver) - else: - raise ValueError('Unsupported RDBMS %s' % driver) - - params = auth.copy() - params['backend'] = driver - return "%(backend)s://%(user)s:%(passwd)s@localhost/%(database)s" % params - - -def drop_database(admin_engine, current_uri): - """Drop temporary database and user after each particular test.""" - - engine = get_engine(current_uri) - driver = engine.name - auth = {'database': engine.url.database, 'user': engine.url.username} - - if driver == 'sqlite': - try: - os.remove(auth['database']) - except OSError: - pass - elif driver in ['mysql', 'postgresql']: - sql = "drop database if exists %(database)s;" - _execute_sql(admin_engine, [sql % auth], driver) - else: - raise ValueError('Unsupported RDBMS %s' % driver) - - -def main(): - """Controller to handle commands - - ::create: Create test user and database with random names. - ::drop: Drop user and database created by previous command. - """ - parser = argparse.ArgumentParser( - description='Controller to handle database creation and dropping' - ' commands.', - epilog='Under normal circumstances is not used directly.' - ' Used in .testr.conf to automate test database creation' - ' and dropping processes.') - subparsers = parser.add_subparsers( - help='Subcommands to manipulate temporary test databases.') - - create = subparsers.add_parser( - 'create', - help='Create temporary test ' - 'databases and users.') - create.set_defaults(which='create') - create.add_argument( - 'instances_count', - type=int, - help='Number of databases to create.') - - drop = subparsers.add_parser( - 'drop', - help='Drop temporary test databases and users.') - drop.set_defaults(which='drop') - drop.add_argument( - 'instances', - nargs='+', - help='List of databases uri to be dropped.') - - args = parser.parse_args() - - connection_string = os.getenv('OS_TEST_DBAPI_ADMIN_CONNECTION', - 'sqlite://') - engine = get_engine(connection_string) - which = args.which - - if which == "create": - for i in range(int(args.instances_count)): - print(create_database(engine)) - elif which == "drop": - for db in args.instances: - drop_database(engine, db) - - -if __name__ == "__main__": - main() diff --git a/logaas/openstack/common/db/sqlalchemy/session.py b/logaas/openstack/common/db/sqlalchemy/session.py deleted file mode 100644 index 268a1a9..0000000 --- a/logaas/openstack/common/db/sqlalchemy/session.py +++ /dev/null @@ -1,904 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Session Handling for SQLAlchemy backend. - -Recommended ways to use sessions within this framework: - -* Don't use them explicitly; this is like running with ``AUTOCOMMIT=1``. - `model_query()` will implicitly use a session when called without one - supplied. This is the ideal situation because it will allow queries - to be automatically retried if the database connection is interrupted. - - .. note:: Automatic retry will be enabled in a future patch. - - It is generally fine to issue several queries in a row like this. Even though - they may be run in separate transactions and/or separate sessions, each one - will see the data from the prior calls. If needed, undo- or rollback-like - functionality should be handled at a logical level. For an example, look at - the code around quotas and `reservation_rollback()`. - - Examples: - - .. code-block:: python - - def get_foo(context, foo): - return (model_query(context, models.Foo). - filter_by(foo=foo). - first()) - - def update_foo(context, id, newfoo): - (model_query(context, models.Foo). - filter_by(id=id). - update({'foo': newfoo})) - - def create_foo(context, values): - foo_ref = models.Foo() - foo_ref.update(values) - foo_ref.save() - return foo_ref - - -* Within the scope of a single method, keep all the reads and writes within - the context managed by a single session. In this way, the session's - `__exit__` handler will take care of calling `flush()` and `commit()` for - you. If using this approach, you should not explicitly call `flush()` or - `commit()`. Any error within the context of the session will cause the - session to emit a `ROLLBACK`. Database errors like `IntegrityError` will be - raised in `session`'s `__exit__` handler, and any try/except within the - context managed by `session` will not be triggered. And catching other - non-database errors in the session will not trigger the ROLLBACK, so - exception handlers should always be outside the session, unless the - developer wants to do a partial commit on purpose. If the connection is - dropped before this is possible, the database will implicitly roll back the - transaction. - - .. note:: Statements in the session scope will not be automatically retried. - - If you create models within the session, they need to be added, but you - do not need to call `model.save()`: - - .. code-block:: python - - def create_many_foo(context, foos): - session = sessionmaker() - with session.begin(): - for foo in foos: - foo_ref = models.Foo() - foo_ref.update(foo) - session.add(foo_ref) - - def update_bar(context, foo_id, newbar): - session = sessionmaker() - with session.begin(): - foo_ref = (model_query(context, models.Foo, session). - filter_by(id=foo_id). - first()) - (model_query(context, models.Bar, session). - filter_by(id=foo_ref['bar_id']). - update({'bar': newbar})) - - .. note:: `update_bar` is a trivially simple example of using - ``with session.begin``. Whereas `create_many_foo` is a good example of - when a transaction is needed, it is always best to use as few queries as - possible. - - The two queries in `update_bar` can be better expressed using a single query - which avoids the need for an explicit transaction. It can be expressed like - so: - - .. code-block:: python - - def update_bar(context, foo_id, newbar): - subq = (model_query(context, models.Foo.id). - filter_by(id=foo_id). - limit(1). - subquery()) - (model_query(context, models.Bar). - filter_by(id=subq.as_scalar()). - update({'bar': newbar})) - - For reference, this emits approximately the following SQL statement: - - .. code-block:: sql - - UPDATE bar SET bar = ${newbar} - WHERE id=(SELECT bar_id FROM foo WHERE id = ${foo_id} LIMIT 1); - - .. note:: `create_duplicate_foo` is a trivially simple example of catching an - exception while using ``with session.begin``. Here create two duplicate - instances with same primary key, must catch the exception out of context - managed by a single session: - - .. code-block:: python - - def create_duplicate_foo(context): - foo1 = models.Foo() - foo2 = models.Foo() - foo1.id = foo2.id = 1 - session = sessionmaker() - try: - with session.begin(): - session.add(foo1) - session.add(foo2) - except exception.DBDuplicateEntry as e: - handle_error(e) - -* Passing an active session between methods. Sessions should only be passed - to private methods. The private method must use a subtransaction; otherwise - SQLAlchemy will throw an error when you call `session.begin()` on an existing - transaction. Public methods should not accept a session parameter and should - not be involved in sessions within the caller's scope. - - Note that this incurs more overhead in SQLAlchemy than the above means - due to nesting transactions, and it is not possible to implicitly retry - failed database operations when using this approach. - - This also makes code somewhat more difficult to read and debug, because a - single database transaction spans more than one method. Error handling - becomes less clear in this situation. When this is needed for code clarity, - it should be clearly documented. - - .. code-block:: python - - def myfunc(foo): - session = sessionmaker() - with session.begin(): - # do some database things - bar = _private_func(foo, session) - return bar - - def _private_func(foo, session=None): - if not session: - session = sessionmaker() - with session.begin(subtransaction=True): - # do some other database things - return bar - - -There are some things which it is best to avoid: - -* Don't keep a transaction open any longer than necessary. - - This means that your ``with session.begin()`` block should be as short - as possible, while still containing all the related calls for that - transaction. - -* Avoid ``with_lockmode('UPDATE')`` when possible. - - In MySQL/InnoDB, when a ``SELECT ... FOR UPDATE`` query does not match - any rows, it will take a gap-lock. This is a form of write-lock on the - "gap" where no rows exist, and prevents any other writes to that space. - This can effectively prevent any INSERT into a table by locking the gap - at the end of the index. Similar problems will occur if the SELECT FOR UPDATE - has an overly broad WHERE clause, or doesn't properly use an index. - - One idea proposed at ODS Fall '12 was to use a normal SELECT to test the - number of rows matching a query, and if only one row is returned, - then issue the SELECT FOR UPDATE. - - The better long-term solution is to use - ``INSERT .. ON DUPLICATE KEY UPDATE``. - However, this can not be done until the "deleted" columns are removed and - proper UNIQUE constraints are added to the tables. - - -Enabling soft deletes: - -* To use/enable soft-deletes, the `SoftDeleteMixin` must be added - to your model class. For example: - - .. code-block:: python - - class NovaBase(models.SoftDeleteMixin, models.ModelBase): - pass - - -Efficient use of soft deletes: - -* There are two possible ways to mark a record as deleted: - `model.soft_delete()` and `query.soft_delete()`. - - The `model.soft_delete()` method works with a single already-fetched entry. - `query.soft_delete()` makes only one db request for all entries that - correspond to the query. - -* In almost all cases you should use `query.soft_delete()`. Some examples: - - .. code-block:: python - - def soft_delete_bar(): - count = model_query(BarModel).find(some_condition).soft_delete() - if count == 0: - raise Exception("0 entries were soft deleted") - - def complex_soft_delete_with_synchronization_bar(session=None): - if session is None: - session = sessionmaker() - with session.begin(subtransactions=True): - count = (model_query(BarModel). - find(some_condition). - soft_delete(synchronize_session=True)) - # Here synchronize_session is required, because we - # don't know what is going on in outer session. - if count == 0: - raise Exception("0 entries were soft deleted") - -* There is only one situation where `model.soft_delete()` is appropriate: when - you fetch a single record, work with it, and mark it as deleted in the same - transaction. - - .. code-block:: python - - def soft_delete_bar_model(): - session = sessionmaker() - with session.begin(): - bar_ref = model_query(BarModel).find(some_condition).first() - # Work with bar_ref - bar_ref.soft_delete(session=session) - - However, if you need to work with all entries that correspond to query and - then soft delete them you should use the `query.soft_delete()` method: - - .. code-block:: python - - def soft_delete_multi_models(): - session = sessionmaker() - with session.begin(): - query = (model_query(BarModel, session=session). - find(some_condition)) - model_refs = query.all() - # Work with model_refs - query.soft_delete(synchronize_session=False) - # synchronize_session=False should be set if there is no outer - # session and these entries are not used after this. - - When working with many rows, it is very important to use query.soft_delete, - which issues a single query. Using `model.soft_delete()`, as in the following - example, is very inefficient. - - .. code-block:: python - - for bar_ref in bar_refs: - bar_ref.soft_delete(session=session) - # This will produce count(bar_refs) db requests. - -""" - -import functools -import logging -import re -import time - -import six -from sqlalchemy import exc as sqla_exc -from sqlalchemy.interfaces import PoolListener -import sqlalchemy.orm -from sqlalchemy.pool import NullPool, StaticPool -from sqlalchemy.sql.expression import literal_column - -from logaas.openstack.common.db import exception -from logaas.openstack.common.gettextutils import _LE, _LW -from logaas.openstack.common import timeutils - - -LOG = logging.getLogger(__name__) - - -class SqliteForeignKeysListener(PoolListener): - """Ensures that the foreign key constraints are enforced in SQLite. - - The foreign key constraints are disabled by default in SQLite, - so the foreign key constraints will be enabled here for every - database connection - """ - def connect(self, dbapi_con, con_record): - dbapi_con.execute('pragma foreign_keys=ON') - - -# note(boris-42): In current versions of DB backends unique constraint -# violation messages follow the structure: -# -# sqlite: -# 1 column - (IntegrityError) column c1 is not unique -# N columns - (IntegrityError) column c1, c2, ..., N are not unique -# -# sqlite since 3.7.16: -# 1 column - (IntegrityError) UNIQUE constraint failed: tbl.k1 -# -# N columns - (IntegrityError) UNIQUE constraint failed: tbl.k1, tbl.k2 -# -# postgres: -# 1 column - (IntegrityError) duplicate key value violates unique -# constraint "users_c1_key" -# N columns - (IntegrityError) duplicate key value violates unique -# constraint "name_of_our_constraint" -# -# mysql: -# 1 column - (IntegrityError) (1062, "Duplicate entry 'value_of_c1' for key -# 'c1'") -# N columns - (IntegrityError) (1062, "Duplicate entry 'values joined -# with -' for key 'name_of_our_constraint'") -# -# ibm_db_sa: -# N columns - (IntegrityError) SQL0803N One or more values in the INSERT -# statement, UPDATE statement, or foreign key update caused by a -# DELETE statement are not valid because the primary key, unique -# constraint or unique index identified by "2" constrains table -# "NOVA.KEY_PAIRS" from having duplicate values for the index -# key. -_DUP_KEY_RE_DB = { - "sqlite": (re.compile(r"^.*columns?([^)]+)(is|are)\s+not\s+unique$"), - re.compile(r"^.*UNIQUE\s+constraint\s+failed:\s+(.+)$")), - "postgresql": (re.compile(r"^.*duplicate\s+key.*\"([^\"]+)\"\s*\n.*$"),), - "mysql": (re.compile(r"^.*\(1062,.*'([^\']+)'\"\)$"),), - "ibm_db_sa": (re.compile(r"^.*SQL0803N.*$"),), -} - - -def _raise_if_duplicate_entry_error(integrity_error, engine_name): - """Raise exception if two entries are duplicated. - - In this function will be raised DBDuplicateEntry exception if integrity - error wrap unique constraint violation. - """ - - def get_columns_from_uniq_cons_or_name(columns): - # note(vsergeyev): UniqueConstraint name convention: "uniq_t0c10c2" - # where `t` it is table name and columns `c1`, `c2` - # are in UniqueConstraint. - uniqbase = "uniq_" - if not columns.startswith(uniqbase): - if engine_name == "postgresql": - return [columns[columns.index("_") + 1:columns.rindex("_")]] - return [columns] - return columns[len(uniqbase):].split("0")[1:] - - if engine_name not in ("ibm_db_sa", "mysql", "sqlite", "postgresql"): - return - - # FIXME(johannes): The usage of the .message attribute has been - # deprecated since Python 2.6. However, the exceptions raised by - # SQLAlchemy can differ when using unicode() and accessing .message. - # An audit across all three supported engines will be necessary to - # ensure there are no regressions. - for pattern in _DUP_KEY_RE_DB[engine_name]: - match = pattern.match(integrity_error.message) - if match: - break - else: - return - - # NOTE(mriedem): The ibm_db_sa integrity error message doesn't provide the - # columns so we have to omit that from the DBDuplicateEntry error. - columns = '' - - if engine_name != 'ibm_db_sa': - columns = match.group(1) - - if engine_name == "sqlite": - columns = [c.split('.')[-1] for c in columns.strip().split(", ")] - else: - columns = get_columns_from_uniq_cons_or_name(columns) - raise exception.DBDuplicateEntry(columns, integrity_error) - - -# NOTE(comstud): In current versions of DB backends, Deadlock violation -# messages follow the structure: -# -# mysql: -# (OperationalError) (1213, 'Deadlock found when trying to get lock; try ' -# 'restarting transaction') -_DEADLOCK_RE_DB = { - "mysql": re.compile(r"^.*\(1213, 'Deadlock.*") -} - - -def _raise_if_deadlock_error(operational_error, engine_name): - """Raise exception on deadlock condition. - - Raise DBDeadlock exception if OperationalError contains a Deadlock - condition. - """ - re = _DEADLOCK_RE_DB.get(engine_name) - if re is None: - return - # FIXME(johannes): The usage of the .message attribute has been - # deprecated since Python 2.6. However, the exceptions raised by - # SQLAlchemy can differ when using unicode() and accessing .message. - # An audit across all three supported engines will be necessary to - # ensure there are no regressions. - m = re.match(operational_error.message) - if not m: - return - raise exception.DBDeadlock(operational_error) - - -def _wrap_db_error(f): - @functools.wraps(f) - def _wrap(self, *args, **kwargs): - try: - assert issubclass( - self.__class__, sqlalchemy.orm.session.Session - ), ('_wrap_db_error() can only be applied to methods of ' - 'subclasses of sqlalchemy.orm.session.Session.') - - return f(self, *args, **kwargs) - except UnicodeEncodeError: - raise exception.DBInvalidUnicodeParameter() - except sqla_exc.OperationalError as e: - _raise_if_db_connection_lost(e, self.bind) - _raise_if_deadlock_error(e, self.bind.dialect.name) - # NOTE(comstud): A lot of code is checking for OperationalError - # so let's not wrap it for now. - raise - # note(boris-42): We should catch unique constraint violation and - # wrap it by our own DBDuplicateEntry exception. Unique constraint - # violation is wrapped by IntegrityError. - except sqla_exc.IntegrityError as e: - # note(boris-42): SqlAlchemy doesn't unify errors from different - # DBs so we must do this. Also in some tables (for example - # instance_types) there are more than one unique constraint. This - # means we should get names of columns, which values violate - # unique constraint, from error message. - _raise_if_duplicate_entry_error(e, self.bind.dialect.name) - raise exception.DBError(e) - except Exception as e: - LOG.exception(_LE('DB exception wrapped.')) - raise exception.DBError(e) - return _wrap - - -def _synchronous_switch_listener(dbapi_conn, connection_rec): - """Switch sqlite connections to non-synchronous mode.""" - dbapi_conn.execute("PRAGMA synchronous = OFF") - - -def _add_regexp_listener(dbapi_con, con_record): - """Add REGEXP function to sqlite connections.""" - - def regexp(expr, item): - reg = re.compile(expr) - return reg.search(six.text_type(item)) is not None - dbapi_con.create_function('regexp', 2, regexp) - - -def _thread_yield(dbapi_con, con_record): - """Ensure other greenthreads get a chance to be executed. - - If we use eventlet.monkey_patch(), eventlet.greenthread.sleep(0) will - execute instead of time.sleep(0). - Force a context switch. With common database backends (eg MySQLdb and - sqlite), there is no implicit yield caused by network I/O since they are - implemented by C libraries that eventlet cannot monkey patch. - """ - time.sleep(0) - - -def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy): - """Ensures that MySQL, PostgreSQL or DB2 connections are alive. - - Borrowed from: - http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f - """ - cursor = dbapi_conn.cursor() - try: - ping_sql = 'select 1' - if engine.name == 'ibm_db_sa': - # DB2 requires a table expression - ping_sql = 'select 1 from (values (1)) AS t1' - cursor.execute(ping_sql) - except Exception as ex: - if engine.dialect.is_disconnect(ex, dbapi_conn, cursor): - msg = _LW('Database server has gone away: %s') % ex - LOG.warning(msg) - - # if the database server has gone away, all connections in the pool - # have become invalid and we can safely close all of them here, - # rather than waste time on checking of every single connection - engine.dispose() - - # this will be handled by SQLAlchemy and will force it to create - # a new connection and retry the original action - raise sqla_exc.DisconnectionError(msg) - else: - raise - - -def _set_session_sql_mode(dbapi_con, connection_rec, sql_mode=None): - """Set the sql_mode session variable. - - MySQL supports several server modes. The default is None, but sessions - may choose to enable server modes like TRADITIONAL, ANSI, - several STRICT_* modes and others. - - Note: passing in '' (empty string) for sql_mode clears - the SQL mode for the session, overriding a potentially set - server default. - """ - - cursor = dbapi_con.cursor() - cursor.execute("SET SESSION sql_mode = %s", [sql_mode]) - - -def _mysql_get_effective_sql_mode(engine): - """Returns the effective SQL mode for connections from the engine pool. - - Returns ``None`` if the mode isn't available, otherwise returns the mode. - - """ - # Get the real effective SQL mode. Even when unset by - # our own config, the server may still be operating in a specific - # SQL mode as set by the server configuration. - # Also note that the checkout listener will be called on execute to - # set the mode if it's registered. - row = engine.execute("SHOW VARIABLES LIKE 'sql_mode'").fetchone() - if row is None: - return - return row[1] - - -def _mysql_check_effective_sql_mode(engine): - """Logs a message based on the effective SQL mode for MySQL connections.""" - realmode = _mysql_get_effective_sql_mode(engine) - - if realmode is None: - LOG.warning(_LW('Unable to detect effective SQL mode')) - return - - LOG.debug('MySQL server mode set to %s', realmode) - # 'TRADITIONAL' mode enables several other modes, so - # we need a substring match here - if not ('TRADITIONAL' in realmode.upper() or - 'STRICT_ALL_TABLES' in realmode.upper()): - LOG.warning(_LW("MySQL SQL mode is '%s', " - "consider enabling TRADITIONAL or STRICT_ALL_TABLES"), - realmode) - - -def _mysql_set_mode_callback(engine, sql_mode): - if sql_mode is not None: - mode_callback = functools.partial(_set_session_sql_mode, - sql_mode=sql_mode) - sqlalchemy.event.listen(engine, 'connect', mode_callback) - _mysql_check_effective_sql_mode(engine) - - -def _is_db_connection_error(args): - """Return True if error in connecting to db.""" - # NOTE(adam_g): This is currently MySQL specific and needs to be extended - # to support Postgres and others. - # For the db2, the error code is -30081 since the db2 is still not ready - conn_err_codes = ('2002', '2003', '2006', '2013', '-30081') - for err_code in conn_err_codes: - if args.find(err_code) != -1: - return True - return False - - -def _raise_if_db_connection_lost(error, engine): - # NOTE(vsergeyev): Function is_disconnect(e, connection, cursor) - # requires connection and cursor in incoming parameters, - # but we have no possibility to create connection if DB - # is not available, so in such case reconnect fails. - # But is_disconnect() ignores these parameters, so it - # makes sense to pass to function None as placeholder - # instead of connection and cursor. - if engine.dialect.is_disconnect(error, None, None): - raise exception.DBConnectionError(error) - - -def create_engine(sql_connection, sqlite_fk=False, mysql_sql_mode=None, - idle_timeout=3600, - connection_debug=0, max_pool_size=None, max_overflow=None, - pool_timeout=None, sqlite_synchronous=True, - connection_trace=False, max_retries=10, retry_interval=10): - """Return a new SQLAlchemy engine.""" - - connection_dict = sqlalchemy.engine.url.make_url(sql_connection) - - engine_args = { - "pool_recycle": idle_timeout, - 'convert_unicode': True, - } - - logger = logging.getLogger('sqlalchemy.engine') - - # Map SQL debug level to Python log level - if connection_debug >= 100: - logger.setLevel(logging.DEBUG) - elif connection_debug >= 50: - logger.setLevel(logging.INFO) - else: - logger.setLevel(logging.WARNING) - - if "sqlite" in connection_dict.drivername: - if sqlite_fk: - engine_args["listeners"] = [SqliteForeignKeysListener()] - engine_args["poolclass"] = NullPool - - if sql_connection == "sqlite://": - engine_args["poolclass"] = StaticPool - engine_args["connect_args"] = {'check_same_thread': False} - else: - if max_pool_size is not None: - engine_args['pool_size'] = max_pool_size - if max_overflow is not None: - engine_args['max_overflow'] = max_overflow - if pool_timeout is not None: - engine_args['pool_timeout'] = pool_timeout - - engine = sqlalchemy.create_engine(sql_connection, **engine_args) - - sqlalchemy.event.listen(engine, 'checkin', _thread_yield) - - if engine.name in ('ibm_db_sa', 'mysql', 'postgresql'): - ping_callback = functools.partial(_ping_listener, engine) - sqlalchemy.event.listen(engine, 'checkout', ping_callback) - if engine.name == 'mysql': - if mysql_sql_mode: - _mysql_set_mode_callback(engine, mysql_sql_mode) - elif 'sqlite' in connection_dict.drivername: - if not sqlite_synchronous: - sqlalchemy.event.listen(engine, 'connect', - _synchronous_switch_listener) - sqlalchemy.event.listen(engine, 'connect', _add_regexp_listener) - - if connection_trace and engine.dialect.dbapi.__name__ == 'MySQLdb': - _patch_mysqldb_with_stacktrace_comments() - - try: - engine.connect() - except sqla_exc.OperationalError as e: - if not _is_db_connection_error(e.args[0]): - raise - - remaining = max_retries - if remaining == -1: - remaining = 'infinite' - while True: - msg = _LW('SQL connection failed. %s attempts left.') - LOG.warning(msg % remaining) - if remaining != 'infinite': - remaining -= 1 - time.sleep(retry_interval) - try: - engine.connect() - break - except sqla_exc.OperationalError as e: - if (remaining != 'infinite' and remaining == 0) or \ - not _is_db_connection_error(e.args[0]): - raise - return engine - - -class Query(sqlalchemy.orm.query.Query): - """Subclass of sqlalchemy.query with soft_delete() method.""" - def soft_delete(self, synchronize_session='evaluate'): - return self.update({'deleted': literal_column('id'), - 'updated_at': literal_column('updated_at'), - 'deleted_at': timeutils.utcnow()}, - synchronize_session=synchronize_session) - - -class Session(sqlalchemy.orm.session.Session): - """Custom Session class to avoid SqlAlchemy Session monkey patching.""" - @_wrap_db_error - def query(self, *args, **kwargs): - return super(Session, self).query(*args, **kwargs) - - @_wrap_db_error - def flush(self, *args, **kwargs): - return super(Session, self).flush(*args, **kwargs) - - @_wrap_db_error - def execute(self, *args, **kwargs): - return super(Session, self).execute(*args, **kwargs) - - -def get_maker(engine, autocommit=True, expire_on_commit=False): - """Return a SQLAlchemy sessionmaker using the given engine.""" - return sqlalchemy.orm.sessionmaker(bind=engine, - class_=Session, - autocommit=autocommit, - expire_on_commit=expire_on_commit, - query_cls=Query) - - -def _patch_mysqldb_with_stacktrace_comments(): - """Adds current stack trace as a comment in queries. - - Patches MySQLdb.cursors.BaseCursor._do_query. - """ - import MySQLdb.cursors - import traceback - - old_mysql_do_query = MySQLdb.cursors.BaseCursor._do_query - - def _do_query(self, q): - stack = '' - for filename, line, method, function in traceback.extract_stack(): - # exclude various common things from trace - if filename.endswith('session.py') and method == '_do_query': - continue - if filename.endswith('api.py') and method == 'wrapper': - continue - if filename.endswith('utils.py') and method == '_inner': - continue - if filename.endswith('exception.py') and method == '_wrap': - continue - # db/api is just a wrapper around db/sqlalchemy/api - if filename.endswith('db/api.py'): - continue - # only trace inside logaas - index = filename.rfind('logaas') - if index == -1: - continue - stack += "File:%s:%s Method:%s() Line:%s | " \ - % (filename[index:], line, method, function) - - # strip trailing " | " from stack - if stack: - stack = stack[:-3] - qq = "%s /* %s */" % (q, stack) - else: - qq = q - old_mysql_do_query(self, qq) - - setattr(MySQLdb.cursors.BaseCursor, '_do_query', _do_query) - - -class EngineFacade(object): - """A helper class for removing of global engine instances from logaas.db. - - As a library, logaas.db can't decide where to store/when to create engine - and sessionmaker instances, so this must be left for a target application. - - On the other hand, in order to simplify the adoption of logaas.db changes, - we'll provide a helper class, which creates engine and sessionmaker - on its instantiation and provides get_engine()/get_session() methods - that are compatible with corresponding utility functions that currently - exist in target projects, e.g. in Nova. - - engine/sessionmaker instances will still be global (and they are meant to - be global), but they will be stored in the app context, rather that in the - logaas.db context. - - Note: using of this helper is completely optional and you are encouraged to - integrate engine/sessionmaker instances into your apps any way you like - (e.g. one might want to bind a session to a request context). Two important - things to remember: - - 1. An Engine instance is effectively a pool of DB connections, so it's - meant to be shared (and it's thread-safe). - 2. A Session instance is not meant to be shared and represents a DB - transactional context (i.e. it's not thread-safe). sessionmaker is - a factory of sessions. - - """ - - def __init__(self, sql_connection, - sqlite_fk=False, autocommit=True, - expire_on_commit=False, **kwargs): - """Initialize engine and sessionmaker instances. - - :param sqlite_fk: enable foreign keys in SQLite - :type sqlite_fk: bool - - :param autocommit: use autocommit mode for created Session instances - :type autocommit: bool - - :param expire_on_commit: expire session objects on commit - :type expire_on_commit: bool - - Keyword arguments: - - :keyword mysql_sql_mode: the SQL mode to be used for MySQL sessions. - (defaults to TRADITIONAL) - :keyword idle_timeout: timeout before idle sql connections are reaped - (defaults to 3600) - :keyword connection_debug: verbosity of SQL debugging information. - 0=None, 100=Everything (defaults to 0) - :keyword max_pool_size: maximum number of SQL connections to keep open - in a pool (defaults to SQLAlchemy settings) - :keyword max_overflow: if set, use this value for max_overflow with - sqlalchemy (defaults to SQLAlchemy settings) - :keyword pool_timeout: if set, use this value for pool_timeout with - sqlalchemy (defaults to SQLAlchemy settings) - :keyword sqlite_synchronous: if True, SQLite uses synchronous mode - (defaults to True) - :keyword connection_trace: add python stack traces to SQL as comment - strings (defaults to False) - :keyword max_retries: maximum db connection retries during startup. - (setting -1 implies an infinite retry count) - (defaults to 10) - :keyword retry_interval: interval between retries of opening a sql - connection (defaults to 10) - - """ - - super(EngineFacade, self).__init__() - - self._engine = create_engine( - sql_connection=sql_connection, - sqlite_fk=sqlite_fk, - mysql_sql_mode=kwargs.get('mysql_sql_mode', 'TRADITIONAL'), - idle_timeout=kwargs.get('idle_timeout', 3600), - connection_debug=kwargs.get('connection_debug', 0), - max_pool_size=kwargs.get('max_pool_size'), - max_overflow=kwargs.get('max_overflow'), - pool_timeout=kwargs.get('pool_timeout'), - sqlite_synchronous=kwargs.get('sqlite_synchronous', True), - connection_trace=kwargs.get('connection_trace', False), - max_retries=kwargs.get('max_retries', 10), - retry_interval=kwargs.get('retry_interval', 10)) - self._session_maker = get_maker( - engine=self._engine, - autocommit=autocommit, - expire_on_commit=expire_on_commit) - - def get_engine(self): - """Get the engine instance (note, that it's shared).""" - - return self._engine - - def get_session(self, **kwargs): - """Get a Session instance. - - If passed, keyword arguments values override the ones used when the - sessionmaker instance was created. - - :keyword autocommit: use autocommit mode for created Session instances - :type autocommit: bool - - :keyword expire_on_commit: expire session objects on commit - :type expire_on_commit: bool - - """ - - for arg in kwargs: - if arg not in ('autocommit', 'expire_on_commit'): - del kwargs[arg] - - return self._session_maker(**kwargs) - - @classmethod - def from_config(cls, connection_string, conf, - sqlite_fk=False, autocommit=True, expire_on_commit=False): - """Initialize EngineFacade using oslo.config config instance options. - - :param connection_string: SQLAlchemy connection string - :type connection_string: string - - :param conf: oslo.config config instance - :type conf: oslo.config.cfg.ConfigOpts - - :param sqlite_fk: enable foreign keys in SQLite - :type sqlite_fk: bool - - :param autocommit: use autocommit mode for created Session instances - :type autocommit: bool - - :param expire_on_commit: expire session objects on commit - :type expire_on_commit: bool - - """ - - return cls(sql_connection=connection_string, - sqlite_fk=sqlite_fk, - autocommit=autocommit, - expire_on_commit=expire_on_commit, - **dict(conf.database.items())) diff --git a/logaas/openstack/common/db/sqlalchemy/test_base.py b/logaas/openstack/common/db/sqlalchemy/test_base.py deleted file mode 100644 index a8212bc..0000000 --- a/logaas/openstack/common/db/sqlalchemy/test_base.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import functools -import os - -import fixtures -from oslotest import base as test_base -import six - -from logaas.openstack.common.db.sqlalchemy import provision -from logaas.openstack.common.db.sqlalchemy import session -from logaas.openstack.common.db.sqlalchemy import utils - - -class DbFixture(fixtures.Fixture): - """Basic database fixture. - - Allows to run tests on various db backends, such as SQLite, MySQL and - PostgreSQL. By default use sqlite backend. To override default backend - uri set env variable OS_TEST_DBAPI_CONNECTION with database admin - credentials for specific backend. - """ - - def _get_uri(self): - return os.getenv('OS_TEST_DBAPI_CONNECTION', 'sqlite://') - - def __init__(self, test): - super(DbFixture, self).__init__() - - self.test = test - - def cleanUp(self): - self.test.engine.dispose() - - def setUp(self): - super(DbFixture, self).setUp() - - self.test.engine = session.create_engine(self._get_uri()) - self.test.sessionmaker = session.get_maker(self.test.engine) - - -class DbTestCase(test_base.BaseTestCase): - """Base class for testing of DB code. - - Using `DbFixture`. Intended to be the main database test case to use all - the tests on a given backend with user defined uri. Backend specific - tests should be decorated with `backend_specific` decorator. - """ - - FIXTURE = DbFixture - - def setUp(self): - super(DbTestCase, self).setUp() - self.useFixture(self.FIXTURE(self)) - - -ALLOWED_DIALECTS = ['sqlite', 'mysql', 'postgresql'] - - -def backend_specific(*dialects): - """Decorator to skip backend specific tests on inappropriate engines. - - ::dialects: list of dialects names under which the test will be launched. - """ - def wrap(f): - @functools.wraps(f) - def ins_wrap(self): - if not set(dialects).issubset(ALLOWED_DIALECTS): - raise ValueError( - "Please use allowed dialects: %s" % ALLOWED_DIALECTS) - if self.engine.name not in dialects: - msg = ('The test "%s" can be run ' - 'only on %s. Current engine is %s.') - args = (f.__name__, ' '.join(dialects), self.engine.name) - self.skip(msg % args) - else: - return f(self) - return ins_wrap - return wrap - - -@six.add_metaclass(abc.ABCMeta) -class OpportunisticFixture(DbFixture): - """Base fixture to use default CI databases. - - The databases exist in OpenStack CI infrastructure. But for the - correct functioning in local environment the databases must be - created manually. - """ - - DRIVER = abc.abstractproperty(lambda: None) - DBNAME = PASSWORD = USERNAME = 'openstack_citest' - - def setUp(self): - self._provisioning_engine = provision.get_engine( - utils.get_connect_string(backend=self.DRIVER, - user=self.USERNAME, - passwd=self.PASSWORD, - database=self.DBNAME) - ) - self._uri = provision.create_database(self._provisioning_engine) - - super(OpportunisticFixture, self).setUp() - - def cleanUp(self): - super(OpportunisticFixture, self).cleanUp() - - provision.drop_database(self._provisioning_engine, self._uri) - - def _get_uri(self): - return self._uri - - -@six.add_metaclass(abc.ABCMeta) -class OpportunisticTestCase(DbTestCase): - """Base test case to use default CI databases. - - The subclasses of the test case are running only when openstack_citest - database is available otherwise a tests will be skipped. - """ - - FIXTURE = abc.abstractproperty(lambda: None) - - def setUp(self): - credentials = { - 'backend': self.FIXTURE.DRIVER, - 'user': self.FIXTURE.USERNAME, - 'passwd': self.FIXTURE.PASSWORD, - 'database': self.FIXTURE.DBNAME} - - if self.FIXTURE.DRIVER and not utils.is_backend_avail(**credentials): - msg = '%s backend is not available.' % self.FIXTURE.DRIVER - return self.skip(msg) - - super(OpportunisticTestCase, self).setUp() - - -class MySQLOpportunisticFixture(OpportunisticFixture): - DRIVER = 'mysql' - - -class PostgreSQLOpportunisticFixture(OpportunisticFixture): - DRIVER = 'postgresql' - - -class MySQLOpportunisticTestCase(OpportunisticTestCase): - FIXTURE = MySQLOpportunisticFixture - - -class PostgreSQLOpportunisticTestCase(OpportunisticTestCase): - FIXTURE = PostgreSQLOpportunisticFixture diff --git a/logaas/openstack/common/db/sqlalchemy/test_migrations.py b/logaas/openstack/common/db/sqlalchemy/test_migrations.py deleted file mode 100644 index 0fa2eb1..0000000 --- a/logaas/openstack/common/db/sqlalchemy/test_migrations.py +++ /dev/null @@ -1,269 +0,0 @@ -# Copyright 2010-2011 OpenStack Foundation -# Copyright 2012-2013 IBM Corp. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import functools -import logging -import os -import subprocess - -import lockfile -from oslotest import base as test_base -from six import moves -from six.moves.urllib import parse -import sqlalchemy -import sqlalchemy.exc - -from logaas.openstack.common.db.sqlalchemy import utils -from logaas.openstack.common.gettextutils import _LE - -LOG = logging.getLogger(__name__) - - -def _have_mysql(user, passwd, database): - present = os.environ.get('TEST_MYSQL_PRESENT') - if present is None: - return utils.is_backend_avail(backend='mysql', - user=user, - passwd=passwd, - database=database) - return present.lower() in ('', 'true') - - -def _have_postgresql(user, passwd, database): - present = os.environ.get('TEST_POSTGRESQL_PRESENT') - if present is None: - return utils.is_backend_avail(backend='postgres', - user=user, - passwd=passwd, - database=database) - return present.lower() in ('', 'true') - - -def _set_db_lock(lock_path=None, lock_prefix=None): - def decorator(f): - @functools.wraps(f) - def wrapper(*args, **kwargs): - try: - path = lock_path or os.environ.get("LOGAAS_LOCK_PATH") - lock = lockfile.FileLock(os.path.join(path, lock_prefix)) - with lock: - LOG.debug('Got lock "%s"' % f.__name__) - return f(*args, **kwargs) - finally: - LOG.debug('Lock released "%s"' % f.__name__) - return wrapper - return decorator - - -class BaseMigrationTestCase(test_base.BaseTestCase): - """Base class fort testing of migration utils.""" - - def __init__(self, *args, **kwargs): - super(BaseMigrationTestCase, self).__init__(*args, **kwargs) - - self.DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), - 'test_migrations.conf') - # Test machines can set the TEST_MIGRATIONS_CONF variable - # to override the location of the config file for migration testing - self.CONFIG_FILE_PATH = os.environ.get('TEST_MIGRATIONS_CONF', - self.DEFAULT_CONFIG_FILE) - self.test_databases = {} - self.migration_api = None - - def setUp(self): - super(BaseMigrationTestCase, self).setUp() - - # Load test databases from the config file. Only do this - # once. No need to re-run this on each test... - LOG.debug('config_path is %s' % self.CONFIG_FILE_PATH) - if os.path.exists(self.CONFIG_FILE_PATH): - cp = moves.configparser.RawConfigParser() - try: - cp.read(self.CONFIG_FILE_PATH) - defaults = cp.defaults() - for key, value in defaults.items(): - self.test_databases[key] = value - except moves.configparser.ParsingError as e: - self.fail("Failed to read test_migrations.conf config " - "file. Got error: %s" % e) - else: - self.fail("Failed to find test_migrations.conf config " - "file.") - - self.engines = {} - for key, value in self.test_databases.items(): - self.engines[key] = sqlalchemy.create_engine(value) - - # We start each test case with a completely blank slate. - self._reset_databases() - - def tearDown(self): - # We destroy the test data store between each test case, - # and recreate it, which ensures that we have no side-effects - # from the tests - self._reset_databases() - super(BaseMigrationTestCase, self).tearDown() - - def execute_cmd(self, cmd=None): - process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - output = process.communicate()[0] - LOG.debug(output) - self.assertEqual(0, process.returncode, - "Failed to run: %s\n%s" % (cmd, output)) - - def _reset_pg(self, conn_pieces): - (user, - password, - database, - host) = utils.get_db_connection_info(conn_pieces) - os.environ['PGPASSWORD'] = password - os.environ['PGUSER'] = user - # note(boris-42): We must create and drop database, we can't - # drop database which we have connected to, so for such - # operations there is a special database template1. - sqlcmd = ("psql -w -U %(user)s -h %(host)s -c" - " '%(sql)s' -d template1") - - sql = ("drop database if exists %s;") % database - droptable = sqlcmd % {'user': user, 'host': host, 'sql': sql} - self.execute_cmd(droptable) - - sql = ("create database %s;") % database - createtable = sqlcmd % {'user': user, 'host': host, 'sql': sql} - self.execute_cmd(createtable) - - os.unsetenv('PGPASSWORD') - os.unsetenv('PGUSER') - - @_set_db_lock(lock_prefix='migration_tests-') - def _reset_databases(self): - for key, engine in self.engines.items(): - conn_string = self.test_databases[key] - conn_pieces = parse.urlparse(conn_string) - engine.dispose() - if conn_string.startswith('sqlite'): - # We can just delete the SQLite database, which is - # the easiest and cleanest solution - db_path = conn_pieces.path.strip('/') - if os.path.exists(db_path): - os.unlink(db_path) - # No need to recreate the SQLite DB. SQLite will - # create it for us if it's not there... - elif conn_string.startswith('mysql'): - # We can execute the MySQL client to destroy and re-create - # the MYSQL database, which is easier and less error-prone - # than using SQLAlchemy to do this via MetaData...trust me. - (user, password, database, host) = \ - utils.get_db_connection_info(conn_pieces) - sql = ("drop database if exists %(db)s; " - "create database %(db)s;") % {'db': database} - cmd = ("mysql -u \"%(user)s\" -p\"%(password)s\" -h %(host)s " - "-e \"%(sql)s\"") % {'user': user, 'password': password, - 'host': host, 'sql': sql} - self.execute_cmd(cmd) - elif conn_string.startswith('postgresql'): - self._reset_pg(conn_pieces) - - -class WalkVersionsMixin(object): - def _walk_versions(self, engine=None, snake_walk=False, downgrade=True): - # Determine latest version script from the repo, then - # upgrade from 1 through to the latest, with no data - # in the databases. This just checks that the schema itself - # upgrades successfully. - - # Place the database under version control - self.migration_api.version_control(engine, self.REPOSITORY, - self.INIT_VERSION) - self.assertEqual(self.INIT_VERSION, - self.migration_api.db_version(engine, - self.REPOSITORY)) - - LOG.debug('latest version is %s' % self.REPOSITORY.latest) - versions = range(self.INIT_VERSION + 1, self.REPOSITORY.latest + 1) - - for version in versions: - # upgrade -> downgrade -> upgrade - self._migrate_up(engine, version, with_data=True) - if snake_walk: - downgraded = self._migrate_down( - engine, version - 1, with_data=True) - if downgraded: - self._migrate_up(engine, version) - - if downgrade: - # Now walk it back down to 0 from the latest, testing - # the downgrade paths. - for version in reversed(versions): - # downgrade -> upgrade -> downgrade - downgraded = self._migrate_down(engine, version - 1) - - if snake_walk and downgraded: - self._migrate_up(engine, version) - self._migrate_down(engine, version - 1) - - def _migrate_down(self, engine, version, with_data=False): - try: - self.migration_api.downgrade(engine, self.REPOSITORY, version) - except NotImplementedError: - # NOTE(sirp): some migrations, namely release-level - # migrations, don't support a downgrade. - return False - - self.assertEqual( - version, self.migration_api.db_version(engine, self.REPOSITORY)) - - # NOTE(sirp): `version` is what we're downgrading to (i.e. the 'target' - # version). So if we have any downgrade checks, they need to be run for - # the previous (higher numbered) migration. - if with_data: - post_downgrade = getattr( - self, "_post_downgrade_%03d" % (version + 1), None) - if post_downgrade: - post_downgrade(engine) - - return True - - def _migrate_up(self, engine, version, with_data=False): - """migrate up to a new version of the db. - - We allow for data insertion and post checks at every - migration version with special _pre_upgrade_### and - _check_### functions in the main test. - """ - # NOTE(sdague): try block is here because it's impossible to debug - # where a failed data migration happens otherwise - try: - if with_data: - data = None - pre_upgrade = getattr( - self, "_pre_upgrade_%03d" % version, None) - if pre_upgrade: - data = pre_upgrade(engine) - - self.migration_api.upgrade(engine, self.REPOSITORY, version) - self.assertEqual(version, - self.migration_api.db_version(engine, - self.REPOSITORY)) - if with_data: - check = getattr(self, "_check_%03d" % version, None) - if check: - check(engine, data) - except Exception: - LOG.error(_LE("Failed to migrate to version %s on engine %s") % - (version, engine)) - raise diff --git a/logaas/openstack/common/db/sqlalchemy/utils.py b/logaas/openstack/common/db/sqlalchemy/utils.py deleted file mode 100644 index 1f68d90..0000000 --- a/logaas/openstack/common/db/sqlalchemy/utils.py +++ /dev/null @@ -1,655 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2010-2011 OpenStack Foundation. -# Copyright 2012 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import re - -import sqlalchemy -from sqlalchemy import Boolean -from sqlalchemy import CheckConstraint -from sqlalchemy import Column -from sqlalchemy.engine import reflection -from sqlalchemy.ext.compiler import compiles -from sqlalchemy import func -from sqlalchemy import Index -from sqlalchemy import Integer -from sqlalchemy import MetaData -from sqlalchemy import or_ -from sqlalchemy.sql.expression import literal_column -from sqlalchemy.sql.expression import UpdateBase -from sqlalchemy import String -from sqlalchemy import Table -from sqlalchemy.types import NullType - -from logaas.openstack.common import context as request_context -from logaas.openstack.common.db.sqlalchemy import models -from logaas.openstack.common.gettextutils import _, _LI, _LW -from logaas.openstack.common import timeutils - - -LOG = logging.getLogger(__name__) - -_DBURL_REGEX = re.compile(r"[^:]+://([^:]+):([^@]+)@.+") - - -def sanitize_db_url(url): - match = _DBURL_REGEX.match(url) - if match: - return '%s****:****%s' % (url[:match.start(1)], url[match.end(2):]) - return url - - -class InvalidSortKey(Exception): - message = _("Sort key supplied was not valid.") - - -# copy from glance/db/sqlalchemy/api.py -def paginate_query(query, model, limit, sort_keys, marker=None, - sort_dir=None, sort_dirs=None): - """Returns a query with sorting / pagination criteria added. - - Pagination works by requiring a unique sort_key, specified by sort_keys. - (If sort_keys is not unique, then we risk looping through values.) - We use the last row in the previous page as the 'marker' for pagination. - So we must return values that follow the passed marker in the order. - With a single-valued sort_key, this would be easy: sort_key > X. - With a compound-values sort_key, (k1, k2, k3) we must do this to repeat - the lexicographical ordering: - (k1 > X1) or (k1 == X1 && k2 > X2) or (k1 == X1 && k2 == X2 && k3 > X3) - - We also have to cope with different sort_directions. - - Typically, the id of the last row is used as the client-facing pagination - marker, then the actual marker object must be fetched from the db and - passed in to us as marker. - - :param query: the query object to which we should add paging/sorting - :param model: the ORM model class - :param limit: maximum number of items to return - :param sort_keys: array of attributes by which results should be sorted - :param marker: the last item of the previous page; we returns the next - results after this value. - :param sort_dir: direction in which results should be sorted (asc, desc) - :param sort_dirs: per-column array of sort_dirs, corresponding to sort_keys - - :rtype: sqlalchemy.orm.query.Query - :return: The query with sorting/pagination added. - """ - - if 'id' not in sort_keys: - # TODO(justinsb): If this ever gives a false-positive, check - # the actual primary key, rather than assuming its id - LOG.warning(_LW('Id not in sort_keys; is sort_keys unique?')) - - assert(not (sort_dir and sort_dirs)) - - # Default the sort direction to ascending - if sort_dirs is None and sort_dir is None: - sort_dir = 'asc' - - # Ensure a per-column sort direction - if sort_dirs is None: - sort_dirs = [sort_dir for _sort_key in sort_keys] - - assert(len(sort_dirs) == len(sort_keys)) - - # Add sorting - for current_sort_key, current_sort_dir in zip(sort_keys, sort_dirs): - try: - sort_dir_func = { - 'asc': sqlalchemy.asc, - 'desc': sqlalchemy.desc, - }[current_sort_dir] - except KeyError: - raise ValueError(_("Unknown sort direction, " - "must be 'desc' or 'asc'")) - try: - sort_key_attr = getattr(model, current_sort_key) - except AttributeError: - raise InvalidSortKey() - query = query.order_by(sort_dir_func(sort_key_attr)) - - # Add pagination - if marker is not None: - marker_values = [] - for sort_key in sort_keys: - v = getattr(marker, sort_key) - marker_values.append(v) - - # Build up an array of sort criteria as in the docstring - criteria_list = [] - for i in range(len(sort_keys)): - crit_attrs = [] - for j in range(i): - model_attr = getattr(model, sort_keys[j]) - crit_attrs.append((model_attr == marker_values[j])) - - model_attr = getattr(model, sort_keys[i]) - if sort_dirs[i] == 'desc': - crit_attrs.append((model_attr < marker_values[i])) - else: - crit_attrs.append((model_attr > marker_values[i])) - - criteria = sqlalchemy.sql.and_(*crit_attrs) - criteria_list.append(criteria) - - f = sqlalchemy.sql.or_(*criteria_list) - query = query.filter(f) - - if limit is not None: - query = query.limit(limit) - - return query - - -def _read_deleted_filter(query, db_model, read_deleted): - if 'deleted' not in db_model.__table__.columns: - raise ValueError(_("There is no `deleted` column in `%s` table. " - "Project doesn't use soft-deleted feature.") - % db_model.__name__) - - default_deleted_value = db_model.__table__.c.deleted.default.arg - if read_deleted == 'no': - query = query.filter(db_model.deleted == default_deleted_value) - elif read_deleted == 'yes': - pass # omit the filter to include deleted and active - elif read_deleted == 'only': - query = query.filter(db_model.deleted != default_deleted_value) - else: - raise ValueError(_("Unrecognized read_deleted value '%s'") - % read_deleted) - return query - - -def _project_filter(query, db_model, context, project_only): - if project_only and 'project_id' not in db_model.__table__.columns: - raise ValueError(_("There is no `project_id` column in `%s` table.") - % db_model.__name__) - - if request_context.is_user_context(context) and project_only: - if project_only == 'allow_none': - is_none = None - query = query.filter(or_(db_model.project_id == context.project_id, - db_model.project_id == is_none)) - else: - query = query.filter(db_model.project_id == context.project_id) - - return query - - -def model_query(context, model, session, args=None, project_only=False, - read_deleted=None): - """Query helper that accounts for context's `read_deleted` field. - - :param context: context to query under - - :param model: Model to query. Must be a subclass of ModelBase. - :type model: models.ModelBase - - :param session: The session to use. - :type session: sqlalchemy.orm.session.Session - - :param args: Arguments to query. If None - model is used. - :type args: tuple - - :param project_only: If present and context is user-type, then restrict - query to match the context's project_id. If set to - 'allow_none', restriction includes project_id = None. - :type project_only: bool - - :param read_deleted: If present, overrides context's read_deleted field. - :type read_deleted: bool - - Usage: - - ..code:: python - - result = (utils.model_query(context, models.Instance, session=session) - .filter_by(uuid=instance_uuid) - .all()) - - query = utils.model_query( - context, Node, - session=session, - args=(func.count(Node.id), func.sum(Node.ram)) - ).filter_by(project_id=project_id) - - """ - - if not read_deleted: - if hasattr(context, 'read_deleted'): - # NOTE(viktors): some projects use `read_deleted` attribute in - # their contexts instead of `show_deleted`. - read_deleted = context.read_deleted - else: - read_deleted = context.show_deleted - - if not issubclass(model, models.ModelBase): - raise TypeError(_("model should be a subclass of ModelBase")) - - query = session.query(model) if not args else session.query(*args) - query = _read_deleted_filter(query, model, read_deleted) - query = _project_filter(query, model, context, project_only) - - return query - - -def get_table(engine, name): - """Returns an sqlalchemy table dynamically from db. - - Needed because the models don't work for us in migrations - as models will be far out of sync with the current data. - - .. warning:: - - Do not use this method when creating ForeignKeys in database migrations - because sqlalchemy needs the same MetaData object to hold information - about the parent table and the reference table in the ForeignKey. This - method uses a unique MetaData object per table object so it won't work - with ForeignKey creation. - """ - metadata = MetaData() - metadata.bind = engine - return Table(name, metadata, autoload=True) - - -class InsertFromSelect(UpdateBase): - """Form the base for `INSERT INTO table (SELECT ... )` statement.""" - def __init__(self, table, select): - self.table = table - self.select = select - - -@compiles(InsertFromSelect) -def visit_insert_from_select(element, compiler, **kw): - """Form the `INSERT INTO table (SELECT ... )` statement.""" - return "INSERT INTO %s %s" % ( - compiler.process(element.table, asfrom=True), - compiler.process(element.select)) - - -class ColumnError(Exception): - """Error raised when no column or an invalid column is found.""" - - -def _get_not_supported_column(col_name_col_instance, column_name): - try: - column = col_name_col_instance[column_name] - except KeyError: - msg = _("Please specify column %s in col_name_col_instance " - "param. It is required because column has unsupported " - "type by sqlite).") - raise ColumnError(msg % column_name) - - if not isinstance(column, Column): - msg = _("col_name_col_instance param has wrong type of " - "column instance for column %s It should be instance " - "of sqlalchemy.Column.") - raise ColumnError(msg % column_name) - return column - - -def drop_unique_constraint(migrate_engine, table_name, uc_name, *columns, - **col_name_col_instance): - """Drop unique constraint from table. - - DEPRECATED: this function is deprecated and will be removed from logaas.db - in a few releases. Please use UniqueConstraint.drop() method directly for - sqlalchemy-migrate migration scripts. - - This method drops UC from table and works for mysql, postgresql and sqlite. - In mysql and postgresql we are able to use "alter table" construction. - Sqlalchemy doesn't support some sqlite column types and replaces their - type with NullType in metadata. We process these columns and replace - NullType with the correct column type. - - :param migrate_engine: sqlalchemy engine - :param table_name: name of table that contains uniq constraint. - :param uc_name: name of uniq constraint that will be dropped. - :param columns: columns that are in uniq constraint. - :param col_name_col_instance: contains pair column_name=column_instance. - column_instance is instance of Column. These params - are required only for columns that have unsupported - types by sqlite. For example BigInteger. - """ - - from migrate.changeset import UniqueConstraint - - meta = MetaData() - meta.bind = migrate_engine - t = Table(table_name, meta, autoload=True) - - if migrate_engine.name == "sqlite": - override_cols = [ - _get_not_supported_column(col_name_col_instance, col.name) - for col in t.columns - if isinstance(col.type, NullType) - ] - for col in override_cols: - t.columns.replace(col) - - uc = UniqueConstraint(*columns, table=t, name=uc_name) - uc.drop() - - -def drop_old_duplicate_entries_from_table(migrate_engine, table_name, - use_soft_delete, *uc_column_names): - """Drop all old rows having the same values for columns in uc_columns. - - This method drop (or mark ad `deleted` if use_soft_delete is True) old - duplicate rows form table with name `table_name`. - - :param migrate_engine: Sqlalchemy engine - :param table_name: Table with duplicates - :param use_soft_delete: If True - values will be marked as `deleted`, - if False - values will be removed from table - :param uc_column_names: Unique constraint columns - """ - meta = MetaData() - meta.bind = migrate_engine - - table = Table(table_name, meta, autoload=True) - columns_for_group_by = [table.c[name] for name in uc_column_names] - - columns_for_select = [func.max(table.c.id)] - columns_for_select.extend(columns_for_group_by) - - duplicated_rows_select = sqlalchemy.sql.select( - columns_for_select, group_by=columns_for_group_by, - having=func.count(table.c.id) > 1) - - for row in migrate_engine.execute(duplicated_rows_select): - # NOTE(boris-42): Do not remove row that has the biggest ID. - delete_condition = table.c.id != row[0] - is_none = None # workaround for pyflakes - delete_condition &= table.c.deleted_at == is_none - for name in uc_column_names: - delete_condition &= table.c[name] == row[name] - - rows_to_delete_select = sqlalchemy.sql.select( - [table.c.id]).where(delete_condition) - for row in migrate_engine.execute(rows_to_delete_select).fetchall(): - LOG.info(_LI("Deleting duplicated row with id: %(id)s from table: " - "%(table)s") % dict(id=row[0], table=table_name)) - - if use_soft_delete: - delete_statement = table.update().\ - where(delete_condition).\ - values({ - 'deleted': literal_column('id'), - 'updated_at': literal_column('updated_at'), - 'deleted_at': timeutils.utcnow() - }) - else: - delete_statement = table.delete().where(delete_condition) - migrate_engine.execute(delete_statement) - - -def _get_default_deleted_value(table): - if isinstance(table.c.id.type, Integer): - return 0 - if isinstance(table.c.id.type, String): - return "" - raise ColumnError(_("Unsupported id columns type")) - - -def _restore_indexes_on_deleted_columns(migrate_engine, table_name, indexes): - table = get_table(migrate_engine, table_name) - - insp = reflection.Inspector.from_engine(migrate_engine) - real_indexes = insp.get_indexes(table_name) - existing_index_names = dict( - [(index['name'], index['column_names']) for index in real_indexes]) - - # NOTE(boris-42): Restore indexes on `deleted` column - for index in indexes: - if 'deleted' not in index['column_names']: - continue - name = index['name'] - if name in existing_index_names: - column_names = [table.c[c] for c in existing_index_names[name]] - old_index = Index(name, *column_names, unique=index["unique"]) - old_index.drop(migrate_engine) - - column_names = [table.c[c] for c in index['column_names']] - new_index = Index(index["name"], *column_names, unique=index["unique"]) - new_index.create(migrate_engine) - - -def change_deleted_column_type_to_boolean(migrate_engine, table_name, - **col_name_col_instance): - if migrate_engine.name == "sqlite": - return _change_deleted_column_type_to_boolean_sqlite( - migrate_engine, table_name, **col_name_col_instance) - insp = reflection.Inspector.from_engine(migrate_engine) - indexes = insp.get_indexes(table_name) - - table = get_table(migrate_engine, table_name) - - old_deleted = Column('old_deleted', Boolean, default=False) - old_deleted.create(table, populate_default=False) - - table.update().\ - where(table.c.deleted == table.c.id).\ - values(old_deleted=True).\ - execute() - - table.c.deleted.drop() - table.c.old_deleted.alter(name="deleted") - - _restore_indexes_on_deleted_columns(migrate_engine, table_name, indexes) - - -def _change_deleted_column_type_to_boolean_sqlite(migrate_engine, table_name, - **col_name_col_instance): - insp = reflection.Inspector.from_engine(migrate_engine) - table = get_table(migrate_engine, table_name) - - columns = [] - for column in table.columns: - column_copy = None - if column.name != "deleted": - if isinstance(column.type, NullType): - column_copy = _get_not_supported_column(col_name_col_instance, - column.name) - else: - column_copy = column.copy() - else: - column_copy = Column('deleted', Boolean, default=0) - columns.append(column_copy) - - constraints = [constraint.copy() for constraint in table.constraints] - - meta = table.metadata - new_table = Table(table_name + "__tmp__", meta, - *(columns + constraints)) - new_table.create() - - indexes = [] - for index in insp.get_indexes(table_name): - column_names = [new_table.c[c] for c in index['column_names']] - indexes.append(Index(index["name"], *column_names, - unique=index["unique"])) - - c_select = [] - for c in table.c: - if c.name != "deleted": - c_select.append(c) - else: - c_select.append(table.c.deleted == table.c.id) - - ins = InsertFromSelect(new_table, sqlalchemy.sql.select(c_select)) - migrate_engine.execute(ins) - - table.drop() - [index.create(migrate_engine) for index in indexes] - - new_table.rename(table_name) - new_table.update().\ - where(new_table.c.deleted == new_table.c.id).\ - values(deleted=True).\ - execute() - - -def change_deleted_column_type_to_id_type(migrate_engine, table_name, - **col_name_col_instance): - if migrate_engine.name == "sqlite": - return _change_deleted_column_type_to_id_type_sqlite( - migrate_engine, table_name, **col_name_col_instance) - insp = reflection.Inspector.from_engine(migrate_engine) - indexes = insp.get_indexes(table_name) - - table = get_table(migrate_engine, table_name) - - new_deleted = Column('new_deleted', table.c.id.type, - default=_get_default_deleted_value(table)) - new_deleted.create(table, populate_default=True) - - deleted = True # workaround for pyflakes - table.update().\ - where(table.c.deleted == deleted).\ - values(new_deleted=table.c.id).\ - execute() - table.c.deleted.drop() - table.c.new_deleted.alter(name="deleted") - - _restore_indexes_on_deleted_columns(migrate_engine, table_name, indexes) - - -def _change_deleted_column_type_to_id_type_sqlite(migrate_engine, table_name, - **col_name_col_instance): - # NOTE(boris-42): sqlaclhemy-migrate can't drop column with check - # constraints in sqlite DB and our `deleted` column has - # 2 check constraints. So there is only one way to remove - # these constraints: - # 1) Create new table with the same columns, constraints - # and indexes. (except deleted column). - # 2) Copy all data from old to new table. - # 3) Drop old table. - # 4) Rename new table to old table name. - insp = reflection.Inspector.from_engine(migrate_engine) - meta = MetaData(bind=migrate_engine) - table = Table(table_name, meta, autoload=True) - default_deleted_value = _get_default_deleted_value(table) - - columns = [] - for column in table.columns: - column_copy = None - if column.name != "deleted": - if isinstance(column.type, NullType): - column_copy = _get_not_supported_column(col_name_col_instance, - column.name) - else: - column_copy = column.copy() - else: - column_copy = Column('deleted', table.c.id.type, - default=default_deleted_value) - columns.append(column_copy) - - def is_deleted_column_constraint(constraint): - # NOTE(boris-42): There is no other way to check is CheckConstraint - # associated with deleted column. - if not isinstance(constraint, CheckConstraint): - return False - sqltext = str(constraint.sqltext) - return (sqltext.endswith("deleted in (0, 1)") or - sqltext.endswith("deleted IN (:deleted_1, :deleted_2)")) - - constraints = [] - for constraint in table.constraints: - if not is_deleted_column_constraint(constraint): - constraints.append(constraint.copy()) - - new_table = Table(table_name + "__tmp__", meta, - *(columns + constraints)) - new_table.create() - - indexes = [] - for index in insp.get_indexes(table_name): - column_names = [new_table.c[c] for c in index['column_names']] - indexes.append(Index(index["name"], *column_names, - unique=index["unique"])) - - ins = InsertFromSelect(new_table, table.select()) - migrate_engine.execute(ins) - - table.drop() - [index.create(migrate_engine) for index in indexes] - - new_table.rename(table_name) - deleted = True # workaround for pyflakes - new_table.update().\ - where(new_table.c.deleted == deleted).\ - values(deleted=new_table.c.id).\ - execute() - - # NOTE(boris-42): Fix value of deleted column: False -> "" or 0. - deleted = False # workaround for pyflakes - new_table.update().\ - where(new_table.c.deleted == deleted).\ - values(deleted=default_deleted_value).\ - execute() - - -def get_connect_string(backend, database, user=None, passwd=None): - """Get database connection - - Try to get a connection with a very specific set of values, if we get - these then we'll run the tests, otherwise they are skipped - """ - args = {'backend': backend, - 'user': user, - 'passwd': passwd, - 'database': database} - if backend == 'sqlite': - template = '%(backend)s:///%(database)s' - else: - template = "%(backend)s://%(user)s:%(passwd)s@localhost/%(database)s" - return template % args - - -def is_backend_avail(backend, database, user=None, passwd=None): - try: - connect_uri = get_connect_string(backend=backend, - database=database, - user=user, - passwd=passwd) - engine = sqlalchemy.create_engine(connect_uri) - connection = engine.connect() - except Exception: - # intentionally catch all to handle exceptions even if we don't - # have any backend code loaded. - return False - else: - connection.close() - engine.dispose() - return True - - -def get_db_connection_info(conn_pieces): - database = conn_pieces.path.strip('/') - loc_pieces = conn_pieces.netloc.split('@') - host = loc_pieces[1] - - auth_pieces = loc_pieces[0].split(':') - user = auth_pieces[0] - password = "" - if len(auth_pieces) > 1: - password = auth_pieces[1].strip() - - return (user, password, database, host) diff --git a/logaas/openstack/common/excutils.py b/logaas/openstack/common/excutils.py deleted file mode 100644 index e6e68bd..0000000 --- a/logaas/openstack/common/excutils.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# Copyright 2012, Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Exception related utilities. -""" - -import logging -import sys -import time -import traceback - -import six - -from logaas.openstack.common.gettextutils import _LE - - -class save_and_reraise_exception(object): - """Save current exception, run some code and then re-raise. - - In some cases the exception context can be cleared, resulting in None - being attempted to be re-raised after an exception handler is run. This - can happen when eventlet switches greenthreads or when running an - exception handler, code raises and catches an exception. In both - cases the exception context will be cleared. - - To work around this, we save the exception state, run handler code, and - then re-raise the original exception. If another exception occurs, the - saved exception is logged and the new exception is re-raised. - - In some cases the caller may not want to re-raise the exception, and - for those circumstances this context provides a reraise flag that - can be used to suppress the exception. For example:: - - except Exception: - with save_and_reraise_exception() as ctxt: - decide_if_need_reraise() - if not should_be_reraised: - ctxt.reraise = False - - If another exception occurs and reraise flag is False, - the saved exception will not be logged. - - If the caller wants to raise new exception during exception handling - he/she sets reraise to False initially with an ability to set it back to - True if needed:: - - except Exception: - with save_and_reraise_exception(reraise=False) as ctxt: - [if statements to determine whether to raise a new exception] - # Not raising a new exception, so reraise - ctxt.reraise = True - """ - def __init__(self, reraise=True): - self.reraise = reraise - - def __enter__(self): - self.type_, self.value, self.tb, = sys.exc_info() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if exc_type is not None: - if self.reraise: - logging.error(_LE('Original exception being dropped: %s'), - traceback.format_exception(self.type_, - self.value, - self.tb)) - return False - if self.reraise: - six.reraise(self.type_, self.value, self.tb) - - -def forever_retry_uncaught_exceptions(infunc): - def inner_func(*args, **kwargs): - last_log_time = 0 - last_exc_message = None - exc_count = 0 - while True: - try: - return infunc(*args, **kwargs) - except Exception as exc: - this_exc_message = six.u(str(exc)) - if this_exc_message == last_exc_message: - exc_count += 1 - else: - exc_count = 1 - # Do not log any more frequently than once a minute unless - # the exception message changes - cur_time = int(time.time()) - if (cur_time - last_log_time > 60 or - this_exc_message != last_exc_message): - logging.exception( - _LE('Unexpected exception occurred %d time(s)... ' - 'retrying.') % exc_count) - last_log_time = cur_time - last_exc_message = this_exc_message - exc_count = 0 - # This should be a very rare event. In case it isn't, do - # a sleep. - time.sleep(1) - return inner_func diff --git a/logaas/openstack/common/fileutils.py b/logaas/openstack/common/fileutils.py deleted file mode 100644 index 33f5e04..0000000 --- a/logaas/openstack/common/fileutils.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import contextlib -import errno -import os -import tempfile - -from logaas.openstack.common import excutils -from logaas.openstack.common import log as logging - -LOG = logging.getLogger(__name__) - -_FILE_CACHE = {} - - -def ensure_tree(path): - """Create a directory (and any ancestor directories required) - - :param path: Directory to create - """ - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST: - if not os.path.isdir(path): - raise - else: - raise - - -def read_cached_file(filename, force_reload=False): - """Read from a file if it has been modified. - - :param force_reload: Whether to reload the file. - :returns: A tuple with a boolean specifying if the data is fresh - or not. - """ - global _FILE_CACHE - - if force_reload and filename in _FILE_CACHE: - del _FILE_CACHE[filename] - - reloaded = False - mtime = os.path.getmtime(filename) - cache_info = _FILE_CACHE.setdefault(filename, {}) - - if not cache_info or mtime > cache_info.get('mtime', 0): - LOG.debug("Reloading cached file %s" % filename) - with open(filename) as fap: - cache_info['data'] = fap.read() - cache_info['mtime'] = mtime - reloaded = True - return (reloaded, cache_info['data']) - - -def delete_if_exists(path, remove=os.unlink): - """Delete a file, but ignore file not found error. - - :param path: File to delete - :param remove: Optional function to remove passed path - """ - - try: - remove(path) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - -@contextlib.contextmanager -def remove_path_on_error(path, remove=delete_if_exists): - """Protect code that wants to operate on PATH atomically. - Any exception will cause PATH to be removed. - - :param path: File to work with - :param remove: Optional function to remove passed path - """ - - try: - yield - except Exception: - with excutils.save_and_reraise_exception(): - remove(path) - - -def file_open(*args, **kwargs): - """Open file - - see built-in file() documentation for more details - - Note: The reason this is kept in a separate module is to easily - be able to provide a stub module that doesn't alter system - state at all (for unit tests) - """ - return open(*args, **kwargs) - - -def write_to_tempfile(content, path=None, suffix='', prefix='tmp'): - """Create temporary file or use existing file. - - This util is needed for creating temporary file with - specified content, suffix and prefix. If path is not None, - it will be used for writing content. If the path doesn't - exist it'll be created. - - :param content: content for temporary file. - :param path: same as parameter 'dir' for mkstemp - :param suffix: same as parameter 'suffix' for mkstemp - :param prefix: same as parameter 'prefix' for mkstemp - - For example: it can be used in database tests for creating - configuration files. - """ - if path: - ensure_tree(path) - - (fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix) - try: - os.write(fd, content) - finally: - os.close(fd) - return path diff --git a/logaas/openstack/common/fixture/__init__.py b/logaas/openstack/common/fixture/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/logaas/openstack/common/fixture/config.py b/logaas/openstack/common/fixture/config.py deleted file mode 100644 index 9489b85..0000000 --- a/logaas/openstack/common/fixture/config.py +++ /dev/null @@ -1,85 +0,0 @@ -# -# Copyright 2013 Mirantis, Inc. -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import fixtures -from oslo.config import cfg -import six - - -class Config(fixtures.Fixture): - """Allows overriding configuration settings for the test. - - `conf` will be reset on cleanup. - - """ - - def __init__(self, conf=cfg.CONF): - self.conf = conf - - def setUp(self): - super(Config, self).setUp() - # NOTE(morganfainberg): unregister must be added to cleanup before - # reset is because cleanup works in reverse order of registered items, - # and a reset must occur before unregistering options can occur. - self.addCleanup(self._unregister_config_opts) - self.addCleanup(self.conf.reset) - self._registered_config_opts = {} - - def config(self, **kw): - """Override configuration values. - - The keyword arguments are the names of configuration options to - override and their values. - - If a `group` argument is supplied, the overrides are applied to - the specified configuration option group, otherwise the overrides - are applied to the ``default`` group. - - """ - - group = kw.pop('group', None) - for k, v in six.iteritems(kw): - self.conf.set_override(k, v, group) - - def _unregister_config_opts(self): - for group in self._registered_config_opts: - self.conf.unregister_opts(self._registered_config_opts[group], - group=group) - - def register_opt(self, opt, group=None): - """Register a single option for the test run. - - Options registered in this manner will automatically be unregistered - during cleanup. - - If a `group` argument is supplied, it will register the new option - to that group, otherwise the option is registered to the ``default`` - group. - """ - self.conf.register_opt(opt, group=group) - self._registered_config_opts.setdefault(group, set()).add(opt) - - def register_opts(self, opts, group=None): - """Register multiple options for the test run. - - This works in the same manner as register_opt() but takes a list of - options as the first argument. All arguments will be registered to the - same group if the ``group`` argument is supplied, otherwise all options - will be registered to the ``default`` group. - """ - for opt in opts: - self.register_opt(opt, group=group) diff --git a/logaas/openstack/common/fixture/lockutils.py b/logaas/openstack/common/fixture/lockutils.py deleted file mode 100644 index 0bcbd72..0000000 --- a/logaas/openstack/common/fixture/lockutils.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import fixtures - -from logaas.openstack.common import lockutils - - -class LockFixture(fixtures.Fixture): - """External locking fixture. - - This fixture is basically an alternative to the synchronized decorator with - the external flag so that tearDowns and addCleanups will be included in - the lock context for locking between tests. The fixture is recommended to - be the first line in a test method, like so:: - - def test_method(self): - self.useFixture(LockFixture) - ... - - or the first line in setUp if all the test methods in the class are - required to be serialized. Something like:: - - class TestCase(testtools.testcase): - def setUp(self): - self.useFixture(LockFixture) - super(TestCase, self).setUp() - ... - - This is because addCleanups are put on a LIFO queue that gets run after the - test method exits. (either by completing or raising an exception) - """ - def __init__(self, name, lock_file_prefix=None): - self.mgr = lockutils.lock(name, lock_file_prefix, True) - - def setUp(self): - super(LockFixture, self).setUp() - self.addCleanup(self.mgr.__exit__, None, None, None) - self.lock = self.mgr.__enter__() diff --git a/logaas/openstack/common/fixture/logging.py b/logaas/openstack/common/fixture/logging.py deleted file mode 100644 index 3823a03..0000000 --- a/logaas/openstack/common/fixture/logging.py +++ /dev/null @@ -1,34 +0,0 @@ -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import fixtures - - -def get_logging_handle_error_fixture(): - """returns a fixture to make logging raise formatting exceptions. - - Usage: - self.useFixture(logging.get_logging_handle_error_fixture()) - """ - return fixtures.MonkeyPatch('logging.Handler.handleError', - _handleError) - - -def _handleError(self, record): - """Monkey patch for logging.Handler.handleError. - - The default handleError just logs the error to stderr but we want - the option of actually raising an exception. - """ - raise diff --git a/logaas/openstack/common/fixture/mockpatch.py b/logaas/openstack/common/fixture/mockpatch.py deleted file mode 100644 index 4ca6bea..0000000 --- a/logaas/openstack/common/fixture/mockpatch.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -############################################################################## -############################################################################## -## -## DO NOT MODIFY THIS FILE -## -## This file is being graduated to the oslotest library. Please make all -## changes there, and only backport critical fixes here. - dhellmann -## -############################################################################## -############################################################################## - -import fixtures -import mock - - -class PatchObject(fixtures.Fixture): - """Deal with code around mock.""" - - def __init__(self, obj, attr, new=mock.DEFAULT, **kwargs): - self.obj = obj - self.attr = attr - self.kwargs = kwargs - self.new = new - - def setUp(self): - super(PatchObject, self).setUp() - _p = mock.patch.object(self.obj, self.attr, self.new, **self.kwargs) - self.mock = _p.start() - self.addCleanup(_p.stop) - - -class Patch(fixtures.Fixture): - - """Deal with code around mock.patch.""" - - def __init__(self, obj, new=mock.DEFAULT, **kwargs): - self.obj = obj - self.kwargs = kwargs - self.new = new - - def setUp(self): - super(Patch, self).setUp() - _p = mock.patch(self.obj, self.new, **self.kwargs) - self.mock = _p.start() - self.addCleanup(_p.stop) diff --git a/logaas/openstack/common/fixture/moxstubout.py b/logaas/openstack/common/fixture/moxstubout.py deleted file mode 100644 index c76c4bd..0000000 --- a/logaas/openstack/common/fixture/moxstubout.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -############################################################################## -############################################################################## -## -## DO NOT MODIFY THIS FILE -## -## This file is being graduated to the oslotest library. Please make all -## changes there, and only backport critical fixes here. - dhellmann -## -############################################################################## -############################################################################## - -import fixtures -from six.moves import mox - - -class MoxStubout(fixtures.Fixture): - """Deal with code around mox and stubout as a fixture.""" - - def setUp(self): - super(MoxStubout, self).setUp() - # emulate some of the mox stuff, we can't use the metaclass - # because it screws with our generators - self.mox = mox.Mox() - self.stubs = self.mox.stubs - self.addCleanup(self.mox.UnsetStubs) - self.addCleanup(self.mox.VerifyAll) diff --git a/logaas/openstack/common/gettextutils.py b/logaas/openstack/common/gettextutils.py deleted file mode 100644 index 836170d..0000000 --- a/logaas/openstack/common/gettextutils.py +++ /dev/null @@ -1,498 +0,0 @@ -# Copyright 2012 Red Hat, Inc. -# Copyright 2013 IBM Corp. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -gettext for openstack-common modules. - -Usual usage in an openstack.common module: - - from logaas.openstack.common.gettextutils import _ -""" - -import copy -import functools -import gettext -import locale -from logging import handlers -import os - -from babel import localedata -import six - -_AVAILABLE_LANGUAGES = {} - -# FIXME(dhellmann): Remove this when moving to oslo.i18n. -USE_LAZY = False - - -class TranslatorFactory(object): - """Create translator functions - """ - - def __init__(self, domain, lazy=False, localedir=None): - """Establish a set of translation functions for the domain. - - :param domain: Name of translation domain, - specifying a message catalog. - :type domain: str - :param lazy: Delays translation until a message is emitted. - Defaults to False. - :type lazy: Boolean - :param localedir: Directory with translation catalogs. - :type localedir: str - """ - self.domain = domain - self.lazy = lazy - if localedir is None: - localedir = os.environ.get(domain.upper() + '_LOCALEDIR') - self.localedir = localedir - - def _make_translation_func(self, domain=None): - """Return a new translation function ready for use. - - Takes into account whether or not lazy translation is being - done. - - The domain can be specified to override the default from the - factory, but the localedir from the factory is always used - because we assume the log-level translation catalogs are - installed in the same directory as the main application - catalog. - - """ - if domain is None: - domain = self.domain - if self.lazy: - return functools.partial(Message, domain=domain) - t = gettext.translation( - domain, - localedir=self.localedir, - fallback=True, - ) - if six.PY3: - return t.gettext - return t.ugettext - - @property - def primary(self): - "The default translation function." - return self._make_translation_func() - - def _make_log_translation_func(self, level): - return self._make_translation_func(self.domain + '-log-' + level) - - @property - def log_info(self): - "Translate info-level log messages." - return self._make_log_translation_func('info') - - @property - def log_warning(self): - "Translate warning-level log messages." - return self._make_log_translation_func('warning') - - @property - def log_error(self): - "Translate error-level log messages." - return self._make_log_translation_func('error') - - @property - def log_critical(self): - "Translate critical-level log messages." - return self._make_log_translation_func('critical') - - -# NOTE(dhellmann): When this module moves out of the incubator into -# oslo.i18n, these global variables can be moved to an integration -# module within each application. - -# Create the global translation functions. -_translators = TranslatorFactory('logaas') - -# The primary translation function using the well-known name "_" -_ = _translators.primary - -# Translators for log levels. -# -# The abbreviated names are meant to reflect the usual use of a short -# name like '_'. The "L" is for "log" and the other letter comes from -# the level. -_LI = _translators.log_info -_LW = _translators.log_warning -_LE = _translators.log_error -_LC = _translators.log_critical - -# NOTE(dhellmann): End of globals that will move to the application's -# integration module. - - -def enable_lazy(): - """Convenience function for configuring _() to use lazy gettext - - Call this at the start of execution to enable the gettextutils._ - function to use lazy gettext functionality. This is useful if - your project is importing _ directly instead of using the - gettextutils.install() way of importing the _ function. - """ - # FIXME(dhellmann): This function will be removed in oslo.i18n, - # because the TranslatorFactory makes it superfluous. - global _, _LI, _LW, _LE, _LC, USE_LAZY - tf = TranslatorFactory('logaas', lazy=True) - _ = tf.primary - _LI = tf.log_info - _LW = tf.log_warning - _LE = tf.log_error - _LC = tf.log_critical - USE_LAZY = True - - -def install(domain, lazy=False): - """Install a _() function using the given translation domain. - - Given a translation domain, install a _() function using gettext's - install() function. - - The main difference from gettext.install() is that we allow - overriding the default localedir (e.g. /usr/share/locale) using - a translation-domain-specific environment variable (e.g. - NOVA_LOCALEDIR). - - :param domain: the translation domain - :param lazy: indicates whether or not to install the lazy _() function. - The lazy _() introduces a way to do deferred translation - of messages by installing a _ that builds Message objects, - instead of strings, which can then be lazily translated into - any available locale. - """ - if lazy: - from six import moves - tf = TranslatorFactory(domain, lazy=True) - moves.builtins.__dict__['_'] = tf.primary - else: - localedir = '%s_LOCALEDIR' % domain.upper() - if six.PY3: - gettext.install(domain, - localedir=os.environ.get(localedir)) - else: - gettext.install(domain, - localedir=os.environ.get(localedir), - unicode=True) - - -class Message(six.text_type): - """A Message object is a unicode object that can be translated. - - Translation of Message is done explicitly using the translate() method. - For all non-translation intents and purposes, a Message is simply unicode, - and can be treated as such. - """ - - def __new__(cls, msgid, msgtext=None, params=None, - domain='logaas', *args): - """Create a new Message object. - - In order for translation to work gettext requires a message ID, this - msgid will be used as the base unicode text. It is also possible - for the msgid and the base unicode text to be different by passing - the msgtext parameter. - """ - # If the base msgtext is not given, we use the default translation - # of the msgid (which is in English) just in case the system locale is - # not English, so that the base text will be in that locale by default. - if not msgtext: - msgtext = Message._translate_msgid(msgid, domain) - # We want to initialize the parent unicode with the actual object that - # would have been plain unicode if 'Message' was not enabled. - msg = super(Message, cls).__new__(cls, msgtext) - msg.msgid = msgid - msg.domain = domain - msg.params = params - return msg - - def translate(self, desired_locale=None): - """Translate this message to the desired locale. - - :param desired_locale: The desired locale to translate the message to, - if no locale is provided the message will be - translated to the system's default locale. - - :returns: the translated message in unicode - """ - - translated_message = Message._translate_msgid(self.msgid, - self.domain, - desired_locale) - if self.params is None: - # No need for more translation - return translated_message - - # This Message object may have been formatted with one or more - # Message objects as substitution arguments, given either as a single - # argument, part of a tuple, or as one or more values in a dictionary. - # When translating this Message we need to translate those Messages too - translated_params = _translate_args(self.params, desired_locale) - - translated_message = translated_message % translated_params - - return translated_message - - @staticmethod - def _translate_msgid(msgid, domain, desired_locale=None): - if not desired_locale: - system_locale = locale.getdefaultlocale() - # If the system locale is not available to the runtime use English - if not system_locale[0]: - desired_locale = 'en_US' - else: - desired_locale = system_locale[0] - - locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR') - lang = gettext.translation(domain, - localedir=locale_dir, - languages=[desired_locale], - fallback=True) - if six.PY3: - translator = lang.gettext - else: - translator = lang.ugettext - - translated_message = translator(msgid) - return translated_message - - def __mod__(self, other): - # When we mod a Message we want the actual operation to be performed - # by the parent class (i.e. unicode()), the only thing we do here is - # save the original msgid and the parameters in case of a translation - params = self._sanitize_mod_params(other) - unicode_mod = super(Message, self).__mod__(params) - modded = Message(self.msgid, - msgtext=unicode_mod, - params=params, - domain=self.domain) - return modded - - def _sanitize_mod_params(self, other): - """Sanitize the object being modded with this Message. - - - Add support for modding 'None' so translation supports it - - Trim the modded object, which can be a large dictionary, to only - those keys that would actually be used in a translation - - Snapshot the object being modded, in case the message is - translated, it will be used as it was when the Message was created - """ - if other is None: - params = (other,) - elif isinstance(other, dict): - # Merge the dictionaries - # Copy each item in case one does not support deep copy. - params = {} - if isinstance(self.params, dict): - for key, val in self.params.items(): - params[key] = self._copy_param(val) - for key, val in other.items(): - params[key] = self._copy_param(val) - else: - params = self._copy_param(other) - return params - - def _copy_param(self, param): - try: - return copy.deepcopy(param) - except Exception: - # Fallback to casting to unicode this will handle the - # python code-like objects that can't be deep-copied - return six.text_type(param) - - def __add__(self, other): - msg = _('Message objects do not support addition.') - raise TypeError(msg) - - def __radd__(self, other): - return self.__add__(other) - - if six.PY2: - def __str__(self): - # NOTE(luisg): Logging in python 2.6 tries to str() log records, - # and it expects specifically a UnicodeError in order to proceed. - msg = _('Message objects do not support str() because they may ' - 'contain non-ascii characters. ' - 'Please use unicode() or translate() instead.') - raise UnicodeError(msg) - - -def get_available_languages(domain): - """Lists the available languages for the given translation domain. - - :param domain: the domain to get languages for - """ - if domain in _AVAILABLE_LANGUAGES: - return copy.copy(_AVAILABLE_LANGUAGES[domain]) - - localedir = '%s_LOCALEDIR' % domain.upper() - find = lambda x: gettext.find(domain, - localedir=os.environ.get(localedir), - languages=[x]) - - # NOTE(mrodden): en_US should always be available (and first in case - # order matters) since our in-line message strings are en_US - language_list = ['en_US'] - # NOTE(luisg): Babel <1.0 used a function called list(), which was - # renamed to locale_identifiers() in >=1.0, the requirements master list - # requires >=0.9.6, uncapped, so defensively work with both. We can remove - # this check when the master list updates to >=1.0, and update all projects - list_identifiers = (getattr(localedata, 'list', None) or - getattr(localedata, 'locale_identifiers')) - locale_identifiers = list_identifiers() - - for i in locale_identifiers: - if find(i) is not None: - language_list.append(i) - - # NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported - # locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they - # are perfectly legitimate locales: - # https://github.com/mitsuhiko/babel/issues/37 - # In Babel 1.3 they fixed the bug and they support these locales, but - # they are still not explicitly "listed" by locale_identifiers(). - # That is why we add the locales here explicitly if necessary so that - # they are listed as supported. - aliases = {'zh': 'zh_CN', - 'zh_Hant_HK': 'zh_HK', - 'zh_Hant': 'zh_TW', - 'fil': 'tl_PH'} - for (locale, alias) in six.iteritems(aliases): - if locale in language_list and alias not in language_list: - language_list.append(alias) - - _AVAILABLE_LANGUAGES[domain] = language_list - return copy.copy(language_list) - - -def translate(obj, desired_locale=None): - """Gets the translated unicode representation of the given object. - - If the object is not translatable it is returned as-is. - If the locale is None the object is translated to the system locale. - - :param obj: the object to translate - :param desired_locale: the locale to translate the message to, if None the - default system locale will be used - :returns: the translated object in unicode, or the original object if - it could not be translated - """ - message = obj - if not isinstance(message, Message): - # If the object to translate is not already translatable, - # let's first get its unicode representation - message = six.text_type(obj) - if isinstance(message, Message): - # Even after unicoding() we still need to check if we are - # running with translatable unicode before translating - return message.translate(desired_locale) - return obj - - -def _translate_args(args, desired_locale=None): - """Translates all the translatable elements of the given arguments object. - - This method is used for translating the translatable values in method - arguments which include values of tuples or dictionaries. - If the object is not a tuple or a dictionary the object itself is - translated if it is translatable. - - If the locale is None the object is translated to the system locale. - - :param args: the args to translate - :param desired_locale: the locale to translate the args to, if None the - default system locale will be used - :returns: a new args object with the translated contents of the original - """ - if isinstance(args, tuple): - return tuple(translate(v, desired_locale) for v in args) - if isinstance(args, dict): - translated_dict = {} - for (k, v) in six.iteritems(args): - translated_v = translate(v, desired_locale) - translated_dict[k] = translated_v - return translated_dict - return translate(args, desired_locale) - - -class TranslationHandler(handlers.MemoryHandler): - """Handler that translates records before logging them. - - The TranslationHandler takes a locale and a target logging.Handler object - to forward LogRecord objects to after translating them. This handler - depends on Message objects being logged, instead of regular strings. - - The handler can be configured declaratively in the logging.conf as follows: - - [handlers] - keys = translatedlog, translator - - [handler_translatedlog] - class = handlers.WatchedFileHandler - args = ('/var/log/api-localized.log',) - formatter = context - - [handler_translator] - class = openstack.common.log.TranslationHandler - target = translatedlog - args = ('zh_CN',) - - If the specified locale is not available in the system, the handler will - log in the default locale. - """ - - def __init__(self, locale=None, target=None): - """Initialize a TranslationHandler - - :param locale: locale to use for translating messages - :param target: logging.Handler object to forward - LogRecord objects to after translation - """ - # NOTE(luisg): In order to allow this handler to be a wrapper for - # other handlers, such as a FileHandler, and still be able to - # configure it using logging.conf, this handler has to extend - # MemoryHandler because only the MemoryHandlers' logging.conf - # parsing is implemented such that it accepts a target handler. - handlers.MemoryHandler.__init__(self, capacity=0, target=target) - self.locale = locale - - def setFormatter(self, fmt): - self.target.setFormatter(fmt) - - def emit(self, record): - # We save the message from the original record to restore it - # after translation, so other handlers are not affected by this - original_msg = record.msg - original_args = record.args - - try: - self._translate_and_log_record(record) - finally: - record.msg = original_msg - record.args = original_args - - def _translate_and_log_record(self, record): - record.msg = translate(record.msg, self.locale) - - # In addition to translating the message, we also need to translate - # arguments that were passed to the log method that were not part - # of the main message e.g., log.info(_('Some message %s'), this_one)) - record.args = _translate_args(record.args, self.locale) - - self.target.emit(record) diff --git a/logaas/openstack/common/importutils.py b/logaas/openstack/common/importutils.py deleted file mode 100644 index 90add10..0000000 --- a/logaas/openstack/common/importutils.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Import related utilities and helper functions. -""" - -import sys -import traceback - - -def import_class(import_str): - """Returns a class from a string including module and class.""" - mod_str, _sep, class_str = import_str.rpartition('.') - __import__(mod_str) - try: - return getattr(sys.modules[mod_str], class_str) - except AttributeError: - raise ImportError('Class %s cannot be found (%s)' % - (class_str, - traceback.format_exception(*sys.exc_info()))) - - -def import_object(import_str, *args, **kwargs): - """Import a class and return an instance of it.""" - return import_class(import_str)(*args, **kwargs) - - -def import_object_ns(name_space, import_str, *args, **kwargs): - """Tries to import object from default namespace. - - Imports a class and return an instance of it, first by trying - to find the class in a default namespace, then failing back to - a full path if not found in the default namespace. - """ - import_value = "%s.%s" % (name_space, import_str) - try: - return import_class(import_value)(*args, **kwargs) - except ImportError: - return import_class(import_str)(*args, **kwargs) - - -def import_module(import_str): - """Import a module.""" - __import__(import_str) - return sys.modules[import_str] - - -def import_versioned_module(version, submodule=None): - module = 'logaas.v%s' % version - if submodule: - module = '.'.join((module, submodule)) - return import_module(module) - - -def try_import(import_str, default=None): - """Try to import a module and if it fails return default.""" - try: - return import_module(import_str) - except ImportError: - return default diff --git a/logaas/openstack/common/jsonutils.py b/logaas/openstack/common/jsonutils.py deleted file mode 100644 index 5516e9e..0000000 --- a/logaas/openstack/common/jsonutils.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -''' -JSON related utilities. - -This module provides a few things: - - 1) A handy function for getting an object down to something that can be - JSON serialized. See to_primitive(). - - 2) Wrappers around loads() and dumps(). The dumps() wrapper will - automatically use to_primitive() for you if needed. - - 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson - is available. -''' - - -import codecs -import datetime -import functools -import inspect -import itertools -import sys - -if sys.version_info < (2, 7): - # On Python <= 2.6, json module is not C boosted, so try to use - # simplejson module if available - try: - import simplejson as json - except ImportError: - import json -else: - import json - -import six -import six.moves.xmlrpc_client as xmlrpclib - -from logaas.openstack.common import gettextutils -from logaas.openstack.common import importutils -from logaas.openstack.common import strutils -from logaas.openstack.common import timeutils - -netaddr = importutils.try_import("netaddr") - -_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod, - inspect.isfunction, inspect.isgeneratorfunction, - inspect.isgenerator, inspect.istraceback, inspect.isframe, - inspect.iscode, inspect.isbuiltin, inspect.isroutine, - inspect.isabstract] - -_simple_types = (six.string_types + six.integer_types - + (type(None), bool, float)) - - -def to_primitive(value, convert_instances=False, convert_datetime=True, - level=0, max_depth=3): - """Convert a complex object into primitives. - - Handy for JSON serialization. We can optionally handle instances, - but since this is a recursive function, we could have cyclical - data structures. - - To handle cyclical data structures we could track the actual objects - visited in a set, but not all objects are hashable. Instead we just - track the depth of the object inspections and don't go too deep. - - Therefore, convert_instances=True is lossy ... be aware. - - """ - # handle obvious types first - order of basic types determined by running - # full tests on nova project, resulting in the following counts: - # 572754 - # 460353 - # 379632 - # 274610 - # 199918 - # 114200 - # 51817 - # 26164 - # 6491 - # 283 - # 19 - if isinstance(value, _simple_types): - return value - - if isinstance(value, datetime.datetime): - if convert_datetime: - return timeutils.strtime(value) - else: - return value - - # value of itertools.count doesn't get caught by nasty_type_tests - # and results in infinite loop when list(value) is called. - if type(value) == itertools.count: - return six.text_type(value) - - # FIXME(vish): Workaround for LP bug 852095. Without this workaround, - # tests that raise an exception in a mocked method that - # has a @wrap_exception with a notifier will fail. If - # we up the dependency to 0.5.4 (when it is released) we - # can remove this workaround. - if getattr(value, '__module__', None) == 'mox': - return 'mock' - - if level > max_depth: - return '?' - - # The try block may not be necessary after the class check above, - # but just in case ... - try: - recursive = functools.partial(to_primitive, - convert_instances=convert_instances, - convert_datetime=convert_datetime, - level=level, - max_depth=max_depth) - if isinstance(value, dict): - return dict((k, recursive(v)) for k, v in six.iteritems(value)) - elif isinstance(value, (list, tuple)): - return [recursive(lv) for lv in value] - - # It's not clear why xmlrpclib created their own DateTime type, but - # for our purposes, make it a datetime type which is explicitly - # handled - if isinstance(value, xmlrpclib.DateTime): - value = datetime.datetime(*tuple(value.timetuple())[:6]) - - if convert_datetime and isinstance(value, datetime.datetime): - return timeutils.strtime(value) - elif isinstance(value, gettextutils.Message): - return value.data - elif hasattr(value, 'iteritems'): - return recursive(dict(value.iteritems()), level=level + 1) - elif hasattr(value, '__iter__'): - return recursive(list(value)) - elif convert_instances and hasattr(value, '__dict__'): - # Likely an instance of something. Watch for cycles. - # Ignore class member vars. - return recursive(value.__dict__, level=level + 1) - elif netaddr and isinstance(value, netaddr.IPAddress): - return six.text_type(value) - else: - if any(test(value) for test in _nasty_type_tests): - return six.text_type(value) - return value - except TypeError: - # Class objects are tricky since they may define something like - # __iter__ defined but it isn't callable as list(). - return six.text_type(value) - - -def dumps(value, default=to_primitive, **kwargs): - return json.dumps(value, default=default, **kwargs) - - -def loads(s, encoding='utf-8', **kwargs): - return json.loads(strutils.safe_decode(s, encoding), **kwargs) - - -def load(fp, encoding='utf-8', **kwargs): - return json.load(codecs.getreader(encoding)(fp), **kwargs) - - -try: - import anyjson -except ImportError: - pass -else: - anyjson._modules.append((__name__, 'dumps', TypeError, - 'loads', ValueError, 'load')) - anyjson.force_implementation(__name__) diff --git a/logaas/openstack/common/local.py b/logaas/openstack/common/local.py deleted file mode 100644 index 0819d5b..0000000 --- a/logaas/openstack/common/local.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Local storage of variables using weak references""" - -import threading -import weakref - - -class WeakLocal(threading.local): - def __getattribute__(self, attr): - rval = super(WeakLocal, self).__getattribute__(attr) - if rval: - # NOTE(mikal): this bit is confusing. What is stored is a weak - # reference, not the value itself. We therefore need to lookup - # the weak reference and return the inner value here. - rval = rval() - return rval - - def __setattr__(self, attr, value): - value = weakref.ref(value) - return super(WeakLocal, self).__setattr__(attr, value) - - -# NOTE(mikal): the name "store" should be deprecated in the future -store = WeakLocal() - -# A "weak" store uses weak references and allows an object to fall out of scope -# when it falls out of scope in the code that uses the thread local storage. A -# "strong" store will hold a reference to the object so that it never falls out -# of scope. -weak_store = WeakLocal() -strong_store = threading.local() diff --git a/logaas/openstack/common/lockutils.py b/logaas/openstack/common/lockutils.py deleted file mode 100644 index b43a23f..0000000 --- a/logaas/openstack/common/lockutils.py +++ /dev/null @@ -1,378 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import contextlib -import errno -import fcntl -import functools -import os -import shutil -import subprocess -import sys -import tempfile -import threading -import time -import weakref - -from oslo.config import cfg - -from logaas.openstack.common import fileutils -from logaas.openstack.common.gettextutils import _, _LE, _LI -from logaas.openstack.common import log as logging - - -LOG = logging.getLogger(__name__) - - -util_opts = [ - cfg.BoolOpt('disable_process_locking', default=False, - help='Enables or disables inter-process locks.'), - cfg.StrOpt('lock_path', - default=os.environ.get("LOGAAS_LOCK_PATH"), - help='Directory to use for lock files.') -] - - -CONF = cfg.CONF -CONF.register_opts(util_opts) - - -def set_defaults(lock_path): - cfg.set_defaults(util_opts, lock_path=lock_path) - - -class _FileLock(object): - """Lock implementation which allows multiple locks, working around - issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does - not require any cleanup. Since the lock is always held on a file - descriptor rather than outside of the process, the lock gets dropped - automatically if the process crashes, even if __exit__ is not executed. - - There are no guarantees regarding usage by multiple green threads in a - single process here. This lock works only between processes. Exclusive - access between local threads should be achieved using the semaphores - in the @synchronized decorator. - - Note these locks are released when the descriptor is closed, so it's not - safe to close the file descriptor while another green thread holds the - lock. Just opening and closing the lock file can break synchronisation, - so lock files must be accessed only using this abstraction. - """ - - def __init__(self, name): - self.lockfile = None - self.fname = name - - def acquire(self): - basedir = os.path.dirname(self.fname) - - if not os.path.exists(basedir): - fileutils.ensure_tree(basedir) - LOG.info(_LI('Created lock path: %s'), basedir) - - self.lockfile = open(self.fname, 'w') - - while True: - try: - # Using non-blocking locks since green threads are not - # patched to deal with blocking locking calls. - # Also upon reading the MSDN docs for locking(), it seems - # to have a laughable 10 attempts "blocking" mechanism. - self.trylock() - LOG.debug('Got file lock "%s"', self.fname) - return True - except IOError as e: - if e.errno in (errno.EACCES, errno.EAGAIN): - # external locks synchronise things like iptables - # updates - give it some time to prevent busy spinning - time.sleep(0.01) - else: - raise threading.ThreadError(_("Unable to acquire lock on" - " `%(filename)s` due to" - " %(exception)s") % - { - 'filename': self.fname, - 'exception': e, - }) - - def __enter__(self): - self.acquire() - return self - - def release(self): - try: - self.unlock() - self.lockfile.close() - LOG.debug('Released file lock "%s"', self.fname) - except IOError: - LOG.exception(_LE("Could not release the acquired lock `%s`"), - self.fname) - - def __exit__(self, exc_type, exc_val, exc_tb): - self.release() - - def exists(self): - return os.path.exists(self.fname) - - def trylock(self): - raise NotImplementedError() - - def unlock(self): - raise NotImplementedError() - - -class _WindowsLock(_FileLock): - def trylock(self): - msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1) - - def unlock(self): - msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1) - - -class _FcntlLock(_FileLock): - def trylock(self): - fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) - - def unlock(self): - fcntl.lockf(self.lockfile, fcntl.LOCK_UN) - - -class _PosixLock(object): - def __init__(self, name): - # Hash the name because it's not valid to have POSIX semaphore - # names with things like / in them. Then use base64 to encode - # the digest() instead taking the hexdigest() because the - # result is shorter and most systems can't have shm sempahore - # names longer than 31 characters. - h = hashlib.sha1() - h.update(name.encode('ascii')) - self.name = str((b'/' + base64.urlsafe_b64encode( - h.digest())).decode('ascii')) - - def acquire(self, timeout=None): - self.semaphore = posix_ipc.Semaphore(self.name, - flags=posix_ipc.O_CREAT, - initial_value=1) - self.semaphore.acquire(timeout) - return self - - def __enter__(self): - self.acquire() - return self - - def release(self): - self.semaphore.release() - self.semaphore.close() - - def __exit__(self, exc_type, exc_val, exc_tb): - self.release() - - def exists(self): - try: - semaphore = posix_ipc.Semaphore(self.name) - except posix_ipc.ExistentialError: - return False - else: - semaphore.close() - return True - - -if os.name == 'nt': - import msvcrt - InterProcessLock = _WindowsLock - FileLock = _WindowsLock -else: - import base64 - import hashlib - import posix_ipc - InterProcessLock = _PosixLock - FileLock = _FcntlLock - -_semaphores = weakref.WeakValueDictionary() -_semaphores_lock = threading.Lock() - - -def _get_lock_path(name, lock_file_prefix, lock_path=None): - # NOTE(mikal): the lock name cannot contain directory - # separators - name = name.replace(os.sep, '_') - if lock_file_prefix: - sep = '' if lock_file_prefix.endswith('-') else '-' - name = '%s%s%s' % (lock_file_prefix, sep, name) - - local_lock_path = lock_path or CONF.lock_path - - if not local_lock_path: - # NOTE(bnemec): Create a fake lock path for posix locks so we don't - # unnecessarily raise the RequiredOptError below. - if InterProcessLock is not _PosixLock: - raise cfg.RequiredOptError('lock_path') - local_lock_path = 'posixlock:/' - - return os.path.join(local_lock_path, name) - - -def external_lock(name, lock_file_prefix=None, lock_path=None): - LOG.debug('Attempting to grab external lock "%(lock)s"', - {'lock': name}) - - lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path) - - # NOTE(bnemec): If an explicit lock_path was passed to us then it - # means the caller is relying on file-based locking behavior, so - # we can't use posix locks for those calls. - if lock_path: - return FileLock(lock_file_path) - return InterProcessLock(lock_file_path) - - -def remove_external_lock_file(name, lock_file_prefix=None): - """Remove a external lock file when it's not used anymore - This will be helpful when we have a lot of lock files - """ - with internal_lock(name): - lock_file_path = _get_lock_path(name, lock_file_prefix) - try: - os.remove(lock_file_path) - except OSError: - LOG.info(_LI('Failed to remove file %(file)s'), - {'file': lock_file_path}) - - -def internal_lock(name): - with _semaphores_lock: - try: - sem = _semaphores[name] - except KeyError: - sem = threading.Semaphore() - _semaphores[name] = sem - - LOG.debug('Got semaphore "%(lock)s"', {'lock': name}) - return sem - - -@contextlib.contextmanager -def lock(name, lock_file_prefix=None, external=False, lock_path=None): - """Context based lock - - This function yields a `threading.Semaphore` instance (if we don't use - eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is - True, in which case, it'll yield an InterProcessLock instance. - - :param lock_file_prefix: The lock_file_prefix argument is used to provide - lock files on disk with a meaningful prefix. - - :param external: The external keyword argument denotes whether this lock - should work across multiple processes. This means that if two different - workers both run a method decorated with @synchronized('mylock', - external=True), only one of them will execute at a time. - """ - int_lock = internal_lock(name) - with int_lock: - if external and not CONF.disable_process_locking: - ext_lock = external_lock(name, lock_file_prefix, lock_path) - with ext_lock: - yield ext_lock - else: - yield int_lock - LOG.debug('Released semaphore "%(lock)s"', {'lock': name}) - - -def synchronized(name, lock_file_prefix=None, external=False, lock_path=None): - """Synchronization decorator. - - Decorating a method like so:: - - @synchronized('mylock') - def foo(self, *args): - ... - - ensures that only one thread will execute the foo method at a time. - - Different methods can share the same lock:: - - @synchronized('mylock') - def foo(self, *args): - ... - - @synchronized('mylock') - def bar(self, *args): - ... - - This way only one of either foo or bar can be executing at a time. - """ - - def wrap(f): - @functools.wraps(f) - def inner(*args, **kwargs): - try: - with lock(name, lock_file_prefix, external, lock_path): - LOG.debug('Got semaphore / lock "%(function)s"', - {'function': f.__name__}) - return f(*args, **kwargs) - finally: - LOG.debug('Semaphore / lock released "%(function)s"', - {'function': f.__name__}) - return inner - return wrap - - -def synchronized_with_prefix(lock_file_prefix): - """Partial object generator for the synchronization decorator. - - Redefine @synchronized in each project like so:: - - (in nova/utils.py) - from nova.openstack.common import lockutils - - synchronized = lockutils.synchronized_with_prefix('nova-') - - - (in nova/foo.py) - from nova import utils - - @utils.synchronized('mylock') - def bar(self, *args): - ... - - The lock_file_prefix argument is used to provide lock files on disk with a - meaningful prefix. - """ - - return functools.partial(synchronized, lock_file_prefix=lock_file_prefix) - - -def main(argv): - """Create a dir for locks and pass it to command from arguments - - If you run this: - python -m openstack.common.lockutils python setup.py testr - - a temporary directory will be created for all your locks and passed to all - your tests in an environment variable. The temporary dir will be deleted - afterwards and the return value will be preserved. - """ - - lock_dir = tempfile.mkdtemp() - os.environ["LOGAAS_LOCK_PATH"] = lock_dir - try: - ret_val = subprocess.call(argv[1:]) - finally: - shutil.rmtree(lock_dir, ignore_errors=True) - return ret_val - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/logaas/openstack/common/log.py b/logaas/openstack/common/log.py deleted file mode 100644 index 6d35fb5..0000000 --- a/logaas/openstack/common/log.py +++ /dev/null @@ -1,723 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""OpenStack logging handler. - -This module adds to logging functionality by adding the option to specify -a context object when calling the various log methods. If the context object -is not specified, default formatting is used. Additionally, an instance uuid -may be passed as part of the log message, which is intended to make it easier -for admins to find messages related to a specific instance. - -It also allows setting of formatting information through conf. - -""" - -import inspect -import itertools -import logging -import logging.config -import logging.handlers -import os -import re -import sys -import traceback - -from oslo.config import cfg -import six -from six import moves - -from logaas.openstack.common.gettextutils import _ -from logaas.openstack.common import importutils -from logaas.openstack.common import jsonutils -from logaas.openstack.common import local - - -_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" - -_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] - -# NOTE(ldbragst): Let's build a list of regex objects using the list of -# _SANITIZE_KEYS we already have. This way, we only have to add the new key -# to the list of _SANITIZE_KEYS and we can generate regular expressions -# for XML and JSON automatically. -_SANITIZE_PATTERNS = [] -_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', - r'(<%(key)s>).*?()', - r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', - r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', - r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])' - '.*?([\'"])', - r'(%(key)s\s*--?[A-z]+\s*).*?([\s])'] - -for key in _SANITIZE_KEYS: - for pattern in _FORMAT_PATTERNS: - reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) - _SANITIZE_PATTERNS.append(reg_ex) - - -common_cli_opts = [ - cfg.BoolOpt('debug', - short='d', - default=False, - help='Print debugging output (set logging level to ' - 'DEBUG instead of default WARNING level).'), - cfg.BoolOpt('verbose', - short='v', - default=False, - help='Print more verbose output (set logging level to ' - 'INFO instead of default WARNING level).'), -] - -logging_cli_opts = [ - cfg.StrOpt('log-config-append', - metavar='PATH', - deprecated_name='log-config', - help='The name of a logging configuration file. This file ' - 'is appended to any existing logging configuration ' - 'files. For details about logging configuration files, ' - 'see the Python logging module documentation.'), - cfg.StrOpt('log-format', - metavar='FORMAT', - help='DEPRECATED. ' - 'A logging.Formatter log message format string which may ' - 'use any of the available logging.LogRecord attributes. ' - 'This option is deprecated. Please use ' - 'logging_context_format_string and ' - 'logging_default_format_string instead.'), - cfg.StrOpt('log-date-format', - default=_DEFAULT_LOG_DATE_FORMAT, - metavar='DATE_FORMAT', - help='Format string for %%(asctime)s in log records. ' - 'Default: %(default)s .'), - cfg.StrOpt('log-file', - metavar='PATH', - deprecated_name='logfile', - help='(Optional) Name of log file to output to. ' - 'If no default is set, logging will go to stdout.'), - cfg.StrOpt('log-dir', - deprecated_name='logdir', - help='(Optional) The base directory used for relative ' - '--log-file paths.'), - cfg.BoolOpt('use-syslog', - default=False, - help='Use syslog for logging. ' - 'Existing syslog format is DEPRECATED during I, ' - 'and will change in J to honor RFC5424.'), - cfg.BoolOpt('use-syslog-rfc-format', - # TODO(bogdando) remove or use True after existing - # syslog format deprecation in J - default=False, - help='(Optional) Enables or disables syslog rfc5424 format ' - 'for logging. If enabled, prefixes the MSG part of the ' - 'syslog message with APP-NAME (RFC5424). The ' - 'format without the APP-NAME is deprecated in I, ' - 'and will be removed in J.'), - cfg.StrOpt('syslog-log-facility', - default='LOG_USER', - help='Syslog facility to receive log lines.') -] - -generic_log_opts = [ - cfg.BoolOpt('use_stderr', - default=True, - help='Log output to standard error.') -] - -log_opts = [ - cfg.StrOpt('logging_context_format_string', - default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' - '%(name)s [%(request_id)s %(user_identity)s] ' - '%(instance)s%(message)s', - help='Format string to use for log messages with context.'), - cfg.StrOpt('logging_default_format_string', - default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' - '%(name)s [-] %(instance)s%(message)s', - help='Format string to use for log messages without context.'), - cfg.StrOpt('logging_debug_format_suffix', - default='%(funcName)s %(pathname)s:%(lineno)d', - help='Data to append to log format when level is DEBUG.'), - cfg.StrOpt('logging_exception_prefix', - default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' - '%(instance)s', - help='Prefix each line of exception output with this format.'), - cfg.ListOpt('default_log_levels', - default=[ - 'amqp=WARN', - 'amqplib=WARN', - 'boto=WARN', - 'qpid=WARN', - 'sqlalchemy=WARN', - 'suds=INFO', - 'oslo.messaging=INFO', - 'iso8601=WARN', - 'requests.packages.urllib3.connectionpool=WARN' - ], - help='List of logger=LEVEL pairs.'), - cfg.BoolOpt('publish_errors', - default=False, - help='Enables or disables publication of error events.'), - cfg.BoolOpt('fatal_deprecations', - default=False, - help='Enables or disables fatal status of deprecations.'), - - # NOTE(mikal): there are two options here because sometimes we are handed - # a full instance (and could include more information), and other times we - # are just handed a UUID for the instance. - cfg.StrOpt('instance_format', - default='[instance: %(uuid)s] ', - help='The format for an instance that is passed with the log ' - 'message. '), - cfg.StrOpt('instance_uuid_format', - default='[instance: %(uuid)s] ', - help='The format for an instance UUID that is passed with the ' - 'log message. '), -] - -CONF = cfg.CONF -CONF.register_cli_opts(common_cli_opts) -CONF.register_cli_opts(logging_cli_opts) -CONF.register_opts(generic_log_opts) -CONF.register_opts(log_opts) - -# our new audit level -# NOTE(jkoelker) Since we synthesized an audit level, make the logging -# module aware of it so it acts like other levels. -logging.AUDIT = logging.INFO + 1 -logging.addLevelName(logging.AUDIT, 'AUDIT') - - -try: - NullHandler = logging.NullHandler -except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7 - class NullHandler(logging.Handler): - def handle(self, record): - pass - - def emit(self, record): - pass - - def createLock(self): - self.lock = None - - -def _dictify_context(context): - if context is None: - return None - if not isinstance(context, dict) and getattr(context, 'to_dict', None): - context = context.to_dict() - return context - - -def _get_binary_name(): - return os.path.basename(inspect.stack()[-1][1]) - - -def _get_log_file_path(binary=None): - logfile = CONF.log_file - logdir = CONF.log_dir - - if logfile and not logdir: - return logfile - - if logfile and logdir: - return os.path.join(logdir, logfile) - - if logdir: - binary = binary or _get_binary_name() - return '%s.log' % (os.path.join(logdir, binary),) - - return None - - -def mask_password(message, secret="***"): - """Replace password with 'secret' in message. - - :param message: The string which includes security information. - :param secret: value with which to replace passwords. - :returns: The unicode value of message with the password fields masked. - - For example: - - >>> mask_password("'adminPass' : 'aaaaa'") - "'adminPass' : '***'" - >>> mask_password("'admin_pass' : 'aaaaa'") - "'admin_pass' : '***'" - >>> mask_password('"password" : "aaaaa"') - '"password" : "***"' - >>> mask_password("'original_password' : 'aaaaa'") - "'original_password' : '***'" - >>> mask_password("u'original_password' : u'aaaaa'") - "u'original_password' : u'***'" - """ - message = six.text_type(message) - - # NOTE(ldbragst): Check to see if anything in message contains any key - # specified in _SANITIZE_KEYS, if not then just return the message since - # we don't have to mask any passwords. - if not any(key in message for key in _SANITIZE_KEYS): - return message - - secret = r'\g<1>' + secret + r'\g<2>' - for pattern in _SANITIZE_PATTERNS: - message = re.sub(pattern, secret, message) - return message - - -class BaseLoggerAdapter(logging.LoggerAdapter): - - def audit(self, msg, *args, **kwargs): - self.log(logging.AUDIT, msg, *args, **kwargs) - - -class LazyAdapter(BaseLoggerAdapter): - def __init__(self, name='unknown', version='unknown'): - self._logger = None - self.extra = {} - self.name = name - self.version = version - - @property - def logger(self): - if not self._logger: - self._logger = getLogger(self.name, self.version) - return self._logger - - -class ContextAdapter(BaseLoggerAdapter): - warn = logging.LoggerAdapter.warning - - def __init__(self, logger, project_name, version_string): - self.logger = logger - self.project = project_name - self.version = version_string - self._deprecated_messages_sent = dict() - - @property - def handlers(self): - return self.logger.handlers - - def deprecated(self, msg, *args, **kwargs): - """Call this method when a deprecated feature is used. - - If the system is configured for fatal deprecations then the message - is logged at the 'critical' level and :class:`DeprecatedConfig` will - be raised. - - Otherwise, the message will be logged (once) at the 'warn' level. - - :raises: :class:`DeprecatedConfig` if the system is configured for - fatal deprecations. - - """ - stdmsg = _("Deprecated: %s") % msg - if CONF.fatal_deprecations: - self.critical(stdmsg, *args, **kwargs) - raise DeprecatedConfig(msg=stdmsg) - - # Using a list because a tuple with dict can't be stored in a set. - sent_args = self._deprecated_messages_sent.setdefault(msg, list()) - - if args in sent_args: - # Already logged this message, so don't log it again. - return - - sent_args.append(args) - self.warn(stdmsg, *args, **kwargs) - - def process(self, msg, kwargs): - # NOTE(mrodden): catch any Message/other object and - # coerce to unicode before they can get - # to the python logging and possibly - # cause string encoding trouble - if not isinstance(msg, six.string_types): - msg = six.text_type(msg) - - if 'extra' not in kwargs: - kwargs['extra'] = {} - extra = kwargs['extra'] - - context = kwargs.pop('context', None) - if not context: - context = getattr(local.store, 'context', None) - if context: - extra.update(_dictify_context(context)) - - instance = kwargs.pop('instance', None) - instance_uuid = (extra.get('instance_uuid') or - kwargs.pop('instance_uuid', None)) - instance_extra = '' - if instance: - instance_extra = CONF.instance_format % instance - elif instance_uuid: - instance_extra = (CONF.instance_uuid_format - % {'uuid': instance_uuid}) - extra['instance'] = instance_extra - - extra.setdefault('user_identity', kwargs.pop('user_identity', None)) - - extra['project'] = self.project - extra['version'] = self.version - extra['extra'] = extra.copy() - return msg, kwargs - - -class JSONFormatter(logging.Formatter): - def __init__(self, fmt=None, datefmt=None): - # NOTE(jkoelker) we ignore the fmt argument, but its still there - # since logging.config.fileConfig passes it. - self.datefmt = datefmt - - def formatException(self, ei, strip_newlines=True): - lines = traceback.format_exception(*ei) - if strip_newlines: - lines = [moves.filter( - lambda x: x, - line.rstrip().splitlines()) for line in lines] - lines = list(itertools.chain(*lines)) - return lines - - def format(self, record): - message = {'message': record.getMessage(), - 'asctime': self.formatTime(record, self.datefmt), - 'name': record.name, - 'msg': record.msg, - 'args': record.args, - 'levelname': record.levelname, - 'levelno': record.levelno, - 'pathname': record.pathname, - 'filename': record.filename, - 'module': record.module, - 'lineno': record.lineno, - 'funcname': record.funcName, - 'created': record.created, - 'msecs': record.msecs, - 'relative_created': record.relativeCreated, - 'thread': record.thread, - 'thread_name': record.threadName, - 'process_name': record.processName, - 'process': record.process, - 'traceback': None} - - if hasattr(record, 'extra'): - message['extra'] = record.extra - - if record.exc_info: - message['traceback'] = self.formatException(record.exc_info) - - return jsonutils.dumps(message) - - -def _create_logging_excepthook(product_name): - def logging_excepthook(exc_type, value, tb): - extra = {'exc_info': (exc_type, value, tb)} - getLogger(product_name).critical( - "".join(traceback.format_exception_only(exc_type, value)), - **extra) - return logging_excepthook - - -class LogConfigError(Exception): - - message = _('Error loading logging config %(log_config)s: %(err_msg)s') - - def __init__(self, log_config, err_msg): - self.log_config = log_config - self.err_msg = err_msg - - def __str__(self): - return self.message % dict(log_config=self.log_config, - err_msg=self.err_msg) - - -def _load_log_config(log_config_append): - try: - logging.config.fileConfig(log_config_append, - disable_existing_loggers=False) - except moves.configparser.Error as exc: - raise LogConfigError(log_config_append, six.text_type(exc)) - - -def setup(product_name, version='unknown'): - """Setup logging.""" - if CONF.log_config_append: - _load_log_config(CONF.log_config_append) - else: - _setup_logging_from_conf(product_name, version) - sys.excepthook = _create_logging_excepthook(product_name) - - -def set_defaults(logging_context_format_string): - cfg.set_defaults(log_opts, - logging_context_format_string= - logging_context_format_string) - - -def _find_facility_from_conf(): - facility_names = logging.handlers.SysLogHandler.facility_names - facility = getattr(logging.handlers.SysLogHandler, - CONF.syslog_log_facility, - None) - - if facility is None and CONF.syslog_log_facility in facility_names: - facility = facility_names.get(CONF.syslog_log_facility) - - if facility is None: - valid_facilities = facility_names.keys() - consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON', - 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS', - 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP', - 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3', - 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7'] - valid_facilities.extend(consts) - raise TypeError(_('syslog facility must be one of: %s') % - ', '.join("'%s'" % fac - for fac in valid_facilities)) - - return facility - - -class RFCSysLogHandler(logging.handlers.SysLogHandler): - def __init__(self, *args, **kwargs): - self.binary_name = _get_binary_name() - # Do not use super() unless type(logging.handlers.SysLogHandler) - # is 'type' (Python 2.7). - # Use old style calls, if the type is 'classobj' (Python 2.6) - logging.handlers.SysLogHandler.__init__(self, *args, **kwargs) - - def format(self, record): - # Do not use super() unless type(logging.handlers.SysLogHandler) - # is 'type' (Python 2.7). - # Use old style calls, if the type is 'classobj' (Python 2.6) - msg = logging.handlers.SysLogHandler.format(self, record) - msg = self.binary_name + ' ' + msg - return msg - - -def _setup_logging_from_conf(project, version): - log_root = getLogger(None).logger - for handler in log_root.handlers: - log_root.removeHandler(handler) - - if CONF.use_syslog: - facility = _find_facility_from_conf() - # TODO(bogdando) use the format provided by RFCSysLogHandler - # after existing syslog format deprecation in J - if CONF.use_syslog_rfc_format: - syslog = RFCSysLogHandler(address='/dev/log', - facility=facility) - else: - syslog = logging.handlers.SysLogHandler(address='/dev/log', - facility=facility) - log_root.addHandler(syslog) - - logpath = _get_log_file_path() - if logpath: - filelog = logging.handlers.WatchedFileHandler(logpath) - log_root.addHandler(filelog) - - if CONF.use_stderr: - streamlog = ColorHandler() - log_root.addHandler(streamlog) - - elif not logpath: - # pass sys.stdout as a positional argument - # python2.6 calls the argument strm, in 2.7 it's stream - streamlog = logging.StreamHandler(sys.stdout) - log_root.addHandler(streamlog) - - if CONF.publish_errors: - handler = importutils.import_object( - "logaas.openstack.common.log_handler.PublishErrorsHandler", - logging.ERROR) - log_root.addHandler(handler) - - datefmt = CONF.log_date_format - for handler in log_root.handlers: - # NOTE(alaski): CONF.log_format overrides everything currently. This - # should be deprecated in favor of context aware formatting. - if CONF.log_format: - handler.setFormatter(logging.Formatter(fmt=CONF.log_format, - datefmt=datefmt)) - log_root.info('Deprecated: log_format is now deprecated and will ' - 'be removed in the next release') - else: - handler.setFormatter(ContextFormatter(project=project, - version=version, - datefmt=datefmt)) - - if CONF.debug: - log_root.setLevel(logging.DEBUG) - elif CONF.verbose: - log_root.setLevel(logging.INFO) - else: - log_root.setLevel(logging.WARNING) - - for pair in CONF.default_log_levels: - mod, _sep, level_name = pair.partition('=') - logger = logging.getLogger(mod) - # NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name - # to integer code. - if sys.version_info < (2, 7): - level = logging.getLevelName(level_name) - logger.setLevel(level) - else: - logger.setLevel(level_name) - - -_loggers = {} - - -def getLogger(name='unknown', version='unknown'): - if name not in _loggers: - _loggers[name] = ContextAdapter(logging.getLogger(name), - name, - version) - return _loggers[name] - - -def getLazyLogger(name='unknown', version='unknown'): - """Returns lazy logger. - - Creates a pass-through logger that does not create the real logger - until it is really needed and delegates all calls to the real logger - once it is created. - """ - return LazyAdapter(name, version) - - -class WritableLogger(object): - """A thin wrapper that responds to `write` and logs.""" - - def __init__(self, logger, level=logging.INFO): - self.logger = logger - self.level = level - - def write(self, msg): - self.logger.log(self.level, msg.rstrip()) - - -class ContextFormatter(logging.Formatter): - """A context.RequestContext aware formatter configured through flags. - - The flags used to set format strings are: logging_context_format_string - and logging_default_format_string. You can also specify - logging_debug_format_suffix to append extra formatting if the log level is - debug. - - For information about what variables are available for the formatter see: - http://docs.python.org/library/logging.html#formatter - - If available, uses the context value stored in TLS - local.store.context - - """ - - def __init__(self, *args, **kwargs): - """Initialize ContextFormatter instance - - Takes additional keyword arguments which can be used in the message - format string. - - :keyword project: project name - :type project: string - :keyword version: project version - :type version: string - - """ - - self.project = kwargs.pop('project', 'unknown') - self.version = kwargs.pop('version', 'unknown') - - logging.Formatter.__init__(self, *args, **kwargs) - - def format(self, record): - """Uses contextstring if request_id is set, otherwise default.""" - - # store project info - record.project = self.project - record.version = self.version - - # store request info - context = getattr(local.store, 'context', None) - if context: - d = _dictify_context(context) - for k, v in d.items(): - setattr(record, k, v) - - # NOTE(sdague): default the fancier formatting params - # to an empty string so we don't throw an exception if - # they get used - for key in ('instance', 'color', 'user_identity'): - if key not in record.__dict__: - record.__dict__[key] = '' - - if record.__dict__.get('request_id'): - self._fmt = CONF.logging_context_format_string - else: - self._fmt = CONF.logging_default_format_string - - if (record.levelno == logging.DEBUG and - CONF.logging_debug_format_suffix): - self._fmt += " " + CONF.logging_debug_format_suffix - - # Cache this on the record, Logger will respect our formatted copy - if record.exc_info: - record.exc_text = self.formatException(record.exc_info, record) - return logging.Formatter.format(self, record) - - def formatException(self, exc_info, record=None): - """Format exception output with CONF.logging_exception_prefix.""" - if not record: - return logging.Formatter.formatException(self, exc_info) - - stringbuffer = moves.StringIO() - traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], - None, stringbuffer) - lines = stringbuffer.getvalue().split('\n') - stringbuffer.close() - - if CONF.logging_exception_prefix.find('%(asctime)') != -1: - record.asctime = self.formatTime(record, self.datefmt) - - formatted_lines = [] - for line in lines: - pl = CONF.logging_exception_prefix % record.__dict__ - fl = '%s%s' % (pl, line) - formatted_lines.append(fl) - return '\n'.join(formatted_lines) - - -class ColorHandler(logging.StreamHandler): - LEVEL_COLORS = { - logging.DEBUG: '\033[00;32m', # GREEN - logging.INFO: '\033[00;36m', # CYAN - logging.AUDIT: '\033[01;36m', # BOLD CYAN - logging.WARN: '\033[01;33m', # BOLD YELLOW - logging.ERROR: '\033[01;31m', # BOLD RED - logging.CRITICAL: '\033[01;31m', # BOLD RED - } - - def format(self, record): - record.color = self.LEVEL_COLORS[record.levelno] - return logging.StreamHandler.format(self, record) - - -class DeprecatedConfig(Exception): - message = _("Fatal call to deprecated config: %(msg)s") - - def __init__(self, msg): - super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/logaas/openstack/common/strutils.py b/logaas/openstack/common/strutils.py deleted file mode 100644 index b39262b..0000000 --- a/logaas/openstack/common/strutils.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -System-level utilities and helper functions. -""" - -import math -import re -import sys -import unicodedata - -import six - -from logaas.openstack.common.gettextutils import _ - - -UNIT_PREFIX_EXPONENT = { - 'k': 1, - 'K': 1, - 'Ki': 1, - 'M': 2, - 'Mi': 2, - 'G': 3, - 'Gi': 3, - 'T': 4, - 'Ti': 4, -} -UNIT_SYSTEM_INFO = { - 'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')), - 'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')), -} - -TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes') -FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no') - -SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]") -SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+") - - -def int_from_bool_as_string(subject): - """Interpret a string as a boolean and return either 1 or 0. - - Any string value in: - - ('True', 'true', 'On', 'on', '1') - - is interpreted as a boolean True. - - Useful for JSON-decoded stuff and config file parsing - """ - return bool_from_string(subject) and 1 or 0 - - -def bool_from_string(subject, strict=False, default=False): - """Interpret a string as a boolean. - - A case-insensitive match is performed such that strings matching 't', - 'true', 'on', 'y', 'yes', or '1' are considered True and, when - `strict=False`, anything else returns the value specified by 'default'. - - Useful for JSON-decoded stuff and config file parsing. - - If `strict=True`, unrecognized values, including None, will raise a - ValueError which is useful when parsing values passed in from an API call. - Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. - """ - if not isinstance(subject, six.string_types): - subject = six.text_type(subject) - - lowered = subject.strip().lower() - - if lowered in TRUE_STRINGS: - return True - elif lowered in FALSE_STRINGS: - return False - elif strict: - acceptable = ', '.join( - "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS)) - msg = _("Unrecognized value '%(val)s', acceptable values are:" - " %(acceptable)s") % {'val': subject, - 'acceptable': acceptable} - raise ValueError(msg) - else: - return default - - -def safe_decode(text, incoming=None, errors='strict'): - """Decodes incoming text/bytes string using `incoming` if they're not - already unicode. - - :param incoming: Text's current encoding - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: text or a unicode `incoming` encoded - representation of it. - :raises TypeError: If text is not an instance of str - """ - if not isinstance(text, (six.string_types, six.binary_type)): - raise TypeError("%s can't be decoded" % type(text)) - - if isinstance(text, six.text_type): - return text - - if not incoming: - incoming = (sys.stdin.encoding or - sys.getdefaultencoding()) - - try: - return text.decode(incoming, errors) - except UnicodeDecodeError: - # Note(flaper87) If we get here, it means that - # sys.stdin.encoding / sys.getdefaultencoding - # didn't return a suitable encoding to decode - # text. This happens mostly when global LANG - # var is not set correctly and there's no - # default encoding. In this case, most likely - # python will use ASCII or ANSI encoders as - # default encodings but they won't be capable - # of decoding non-ASCII characters. - # - # Also, UTF-8 is being used since it's an ASCII - # extension. - return text.decode('utf-8', errors) - - -def safe_encode(text, incoming=None, - encoding='utf-8', errors='strict'): - """Encodes incoming text/bytes string using `encoding`. - - If incoming is not specified, text is expected to be encoded with - current python's default encoding. (`sys.getdefaultencoding`) - - :param incoming: Text's current encoding - :param encoding: Expected encoding for text (Default UTF-8) - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: text or a bytestring `encoding` encoded - representation of it. - :raises TypeError: If text is not an instance of str - """ - if not isinstance(text, (six.string_types, six.binary_type)): - raise TypeError("%s can't be encoded" % type(text)) - - if not incoming: - incoming = (sys.stdin.encoding or - sys.getdefaultencoding()) - - if isinstance(text, six.text_type): - return text.encode(encoding, errors) - elif text and encoding != incoming: - # Decode text before encoding it with `encoding` - text = safe_decode(text, incoming, errors) - return text.encode(encoding, errors) - else: - return text - - -def string_to_bytes(text, unit_system='IEC', return_int=False): - """Converts a string into an float representation of bytes. - - The units supported for IEC :: - - Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it) - KB, KiB, MB, MiB, GB, GiB, TB, TiB - - The units supported for SI :: - - kb(it), Mb(it), Gb(it), Tb(it) - kB, MB, GB, TB - - Note that the SI unit system does not support capital letter 'K' - - :param text: String input for bytes size conversion. - :param unit_system: Unit system for byte size conversion. - :param return_int: If True, returns integer representation of text - in bytes. (default: decimal) - :returns: Numerical representation of text in bytes. - :raises ValueError: If text has an invalid value. - - """ - try: - base, reg_ex = UNIT_SYSTEM_INFO[unit_system] - except KeyError: - msg = _('Invalid unit system: "%s"') % unit_system - raise ValueError(msg) - match = reg_ex.match(text) - if match: - magnitude = float(match.group(1)) - unit_prefix = match.group(2) - if match.group(3) in ['b', 'bit']: - magnitude /= 8 - else: - msg = _('Invalid string format: %s') % text - raise ValueError(msg) - if not unit_prefix: - res = magnitude - else: - res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix]) - if return_int: - return int(math.ceil(res)) - return res - - -def to_slug(value, incoming=None, errors="strict"): - """Normalize string. - - Convert to lowercase, remove non-word characters, and convert spaces - to hyphens. - - Inspired by Django's `slugify` filter. - - :param value: Text to slugify - :param incoming: Text's current encoding - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: slugified unicode representation of `value` - :raises TypeError: If text is not an instance of str - """ - value = safe_decode(value, incoming, errors) - # NOTE(aababilov): no need to use safe_(encode|decode) here: - # encodings are always "ascii", error handling is always "ignore" - # and types are always known (first: unicode; second: str) - value = unicodedata.normalize("NFKD", value).encode( - "ascii", "ignore").decode("ascii") - value = SLUGIFY_STRIP_RE.sub("", value).strip().lower() - return SLUGIFY_HYPHENATE_RE.sub("-", value) diff --git a/logaas/openstack/common/test.py b/logaas/openstack/common/test.py deleted file mode 100644 index 94fe23e..0000000 --- a/logaas/openstack/common/test.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -############################################################################## -############################################################################## -## -## DO NOT MODIFY THIS FILE -## -## This file is being graduated to the oslotest library. Please make all -## changes there, and only backport critical fixes here. - dhellmann -## -############################################################################## -############################################################################## - -"""Common utilities used in testing""" - -import logging -import os -import tempfile - -import fixtures -import testtools - -_TRUE_VALUES = ('True', 'true', '1', 'yes') -_LOG_FORMAT = "%(levelname)8s [%(name)s] %(message)s" - - -class BaseTestCase(testtools.TestCase): - - def setUp(self): - super(BaseTestCase, self).setUp() - self._set_timeout() - self._fake_output() - self._fake_logs() - self.useFixture(fixtures.NestedTempfile()) - self.useFixture(fixtures.TempHomeDir()) - self.tempdirs = [] - - def _set_timeout(self): - test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0) - try: - test_timeout = int(test_timeout) - except ValueError: - # If timeout value is invalid do not set a timeout. - test_timeout = 0 - if test_timeout > 0: - self.useFixture(fixtures.Timeout(test_timeout, gentle=True)) - - def _fake_output(self): - if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES: - stdout = self.useFixture(fixtures.StringStream('stdout')).stream - self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout)) - if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES: - stderr = self.useFixture(fixtures.StringStream('stderr')).stream - self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) - - def _fake_logs(self): - if os.environ.get('OS_DEBUG') in _TRUE_VALUES: - level = logging.DEBUG - else: - level = logging.INFO - capture_logs = os.environ.get('OS_LOG_CAPTURE') in _TRUE_VALUES - if capture_logs: - self.useFixture( - fixtures.FakeLogger( - format=_LOG_FORMAT, - level=level, - nuke_handlers=capture_logs, - ) - ) - else: - logging.basicConfig(format=_LOG_FORMAT, level=level) - - def create_tempfiles(self, files, ext='.conf'): - tempfiles = [] - for (basename, contents) in files: - if not os.path.isabs(basename): - (fd, path) = tempfile.mkstemp(prefix=basename, suffix=ext) - else: - path = basename + ext - fd = os.open(path, os.O_CREAT | os.O_WRONLY) - tempfiles.append(path) - try: - os.write(fd, contents) - finally: - os.close(fd) - return tempfiles diff --git a/logaas/openstack/common/timeutils.py b/logaas/openstack/common/timeutils.py deleted file mode 100644 index 52688a0..0000000 --- a/logaas/openstack/common/timeutils.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Time related utilities and helper functions. -""" - -import calendar -import datetime -import time - -import iso8601 -import six - - -# ISO 8601 extended time format with microseconds -_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' -_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' -PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND - - -def isotime(at=None, subsecond=False): - """Stringify time in ISO 8601 format.""" - if not at: - at = utcnow() - st = at.strftime(_ISO8601_TIME_FORMAT - if not subsecond - else _ISO8601_TIME_FORMAT_SUBSECOND) - tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' - st += ('Z' if tz == 'UTC' else tz) - return st - - -def parse_isotime(timestr): - """Parse time from ISO 8601 format.""" - try: - return iso8601.parse_date(timestr) - except iso8601.ParseError as e: - raise ValueError(six.text_type(e)) - except TypeError as e: - raise ValueError(six.text_type(e)) - - -def strtime(at=None, fmt=PERFECT_TIME_FORMAT): - """Returns formatted utcnow.""" - if not at: - at = utcnow() - return at.strftime(fmt) - - -def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): - """Turn a formatted time back into a datetime.""" - return datetime.datetime.strptime(timestr, fmt) - - -def normalize_time(timestamp): - """Normalize time in arbitrary timezone to UTC naive object.""" - offset = timestamp.utcoffset() - if offset is None: - return timestamp - return timestamp.replace(tzinfo=None) - offset - - -def is_older_than(before, seconds): - """Return True if before is older than seconds.""" - if isinstance(before, six.string_types): - before = parse_strtime(before).replace(tzinfo=None) - else: - before = before.replace(tzinfo=None) - - return utcnow() - before > datetime.timedelta(seconds=seconds) - - -def is_newer_than(after, seconds): - """Return True if after is newer than seconds.""" - if isinstance(after, six.string_types): - after = parse_strtime(after).replace(tzinfo=None) - else: - after = after.replace(tzinfo=None) - - return after - utcnow() > datetime.timedelta(seconds=seconds) - - -def utcnow_ts(): - """Timestamp version of our utcnow function.""" - if utcnow.override_time is None: - # NOTE(kgriffs): This is several times faster - # than going through calendar.timegm(...) - return int(time.time()) - - return calendar.timegm(utcnow().timetuple()) - - -def utcnow(): - """Overridable version of utils.utcnow.""" - if utcnow.override_time: - try: - return utcnow.override_time.pop(0) - except AttributeError: - return utcnow.override_time - return datetime.datetime.utcnow() - - -def iso8601_from_timestamp(timestamp): - """Returns a iso8601 formatted date from timestamp.""" - return isotime(datetime.datetime.utcfromtimestamp(timestamp)) - - -utcnow.override_time = None - - -def set_time_override(override_time=None): - """Overrides utils.utcnow. - - Make it return a constant time or a list thereof, one at a time. - - :param override_time: datetime instance or list thereof. If not - given, defaults to the current UTC time. - """ - utcnow.override_time = override_time or datetime.datetime.utcnow() - - -def advance_time_delta(timedelta): - """Advance overridden time using a datetime.timedelta.""" - assert(not utcnow.override_time is None) - try: - for dt in utcnow.override_time: - dt += timedelta - except TypeError: - utcnow.override_time += timedelta - - -def advance_time_seconds(seconds): - """Advance overridden time by seconds.""" - advance_time_delta(datetime.timedelta(0, seconds)) - - -def clear_time_override(): - """Remove the overridden time.""" - utcnow.override_time = None - - -def marshall_now(now=None): - """Make an rpc-safe datetime with microseconds. - - Note: tzinfo is stripped, but not required for relative times. - """ - if not now: - now = utcnow() - return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, - minute=now.minute, second=now.second, - microsecond=now.microsecond) - - -def unmarshall_time(tyme): - """Unmarshall a datetime dict.""" - return datetime.datetime(day=tyme['day'], - month=tyme['month'], - year=tyme['year'], - hour=tyme['hour'], - minute=tyme['minute'], - second=tyme['second'], - microsecond=tyme['microsecond']) - - -def delta_seconds(before, after): - """Return the difference between two timing objects. - - Compute the difference in seconds between two date, time, or - datetime objects (as a float, to microsecond resolution). - """ - delta = after - before - return total_seconds(delta) - - -def total_seconds(delta): - """Return the total seconds of datetime.timedelta object. - - Compute total seconds of datetime.timedelta, datetime.timedelta - doesn't have method total_seconds in Python2.6, calculate it manually. - """ - try: - return delta.total_seconds() - except AttributeError: - return ((delta.days * 24 * 3600) + delta.seconds + - float(delta.microseconds) / (10 ** 6)) - - -def is_soon(dt, window): - """Determines if time is going to happen in the next window seconds. - - :param dt: the time - :param window: minimum seconds to remain to consider the time not soon - - :return: True if expiration is within the given duration - """ - soon = (utcnow() + datetime.timedelta(seconds=window)) - return normalize_time(dt) <= soon diff --git a/logaas/openstack/common/uuidutils.py b/logaas/openstack/common/uuidutils.py deleted file mode 100644 index 234b880..0000000 --- a/logaas/openstack/common/uuidutils.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2012 Intel Corporation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -UUID related utilities and helper functions. -""" - -import uuid - - -def generate_uuid(): - return str(uuid.uuid4()) - - -def is_uuid_like(val): - """Returns validation of a value as a UUID. - - For our purposes, a UUID is a canonical form string: - aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa - - """ - try: - return str(uuid.UUID(val)) == val - except (TypeError, ValueError, AttributeError): - return False diff --git a/logaas/version.py b/logaas/version.py deleted file mode 100644 index 2a7c1aa..0000000 --- a/logaas/version.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from pbr import version as pbr_version - -LOGAAS_VENDOR = "OpenStack Foundation" -LOGAAS_PRODUCT = "OpenStack Logaas" -LOGAAS_PACKAGE = None # OS distro package version suffix - -loaded = False -version_info = pbr_version.VersionInfo('logaas') - - -def version_string(): - return version_info.version_string() diff --git a/openstack-common.conf b/openstack-common.conf deleted file mode 100644 index 29371b5..0000000 --- a/openstack-common.conf +++ /dev/null @@ -1,16 +0,0 @@ -[DEFAULT] - -# The list of modules to copy from oslo-incubator.git -module=cliutils -module=config -module=db -module=db.sqlalchemy -module=gettextutils -module=importutils -module=log -module=test -module=fixture -module=uuidutils - -# The base module to hold the copy of openstack.common -base=logaas diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 60a6702..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -WebOb>=1.2.3 -Babel>=1.3 -fixtures>=0.3.14 -iso8601>=0.1.9 -pbr>=0.6,!=0.7,<1.0 -PrettyTable>=0.7,<0.8 -oslo.config>=1.2.0 -SQLAlchemy>=0.7.8,<=0.9.99 - diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 587d75a..0000000 --- a/setup.cfg +++ /dev/null @@ -1,36 +0,0 @@ -[metadata] -name = logaas -version = 0.0.1 -summary = Logging as a Service -description-file = - README.rst -author = OpenStack -author-email = openstack-dev@lists.openstack.org -home-page = http://www.openstack.org/ -classifier = - Environment :: OpenStack - Intended Audience :: Developers - Intended Audience :: Information Technology - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.6 - Programming Language :: Python :: 2.7 - -[files] -packages = - logaas - -[entry_points] -console_scripts = - logaas-manage = logaas.cmd.manage:main - -[global] -setup-hooks = - pbr.hooks.setup_hook - -[build_sphinx] -all_files = 1 -build-dir = doc/build -source-dir = doc/source \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 7363757..0000000 --- a/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT -import setuptools - -# In python < 2.7.4, a lazy loading of package `pbr` will break -# setuptools if some other modules registered functions in `atexit`. -# solution from: http://bugs.python.org/issue15881#msg170215 -try: - import multiprocessing # noqa -except ImportError: - pass - -setuptools.setup( - setup_requires=['pbr'], - pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 7c4a8c7..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -hacking>=0.9.2,<0.10 - -coverage>=3.6 -discover -mock>=1.0 -python-subunit>=0.0.18 -testrepository>=0.0.18 -testtools>=0.9.34 - -sphinx>=1.1.2,<1.2 -oslosphinx -oslotest diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/cmd/__init__.py b/tests/cmd/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/cmd/test_manage.py b/tests/cmd/test_manage.py deleted file mode 100644 index aadc559..0000000 --- a/tests/cmd/test_manage.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import mock - -from logaas.cmd import manage -from tests import test - - -class DBCommandsTestCase(test.TestCase): - - @mock.patch("logaas.cmd.manage.db") - def test_create(self, mock_db): - manage.DBCommands().create() - mock_db.db_create.assert_called_once_with() - - @mock.patch("logaas.cmd.manage.db") - def test_recreate(self, mock_db): - manage.DBCommands().recreate() - mock_db.db_drop.assert_called_once_with() - mock_db.db_create.assert_called_once_with() diff --git a/tests/db/__init__.py b/tests/db/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/db/test_api.py b/tests/db/test_api.py deleted file mode 100644 index ed5b565..0000000 --- a/tests/db/test_api.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from logaas import db -from logaas.openstack.common.db import options as db_options -from tests import test - - -db_options.set_defaults(sql_connection='sqlite://', sqlite_db='logaas.sqlite') - - -class ManageDBTestCase(test.TestCase): - - def test_cleanup(self): - # TODO(boris-42): Test more detailed when we will have something in db. - db.db_cleanup() - - def test_create(self): - # TODO(boris-42): Test more detailed when we will have something in db. - db.db_create() - - def test_create_delete_create(self): - # TODO(boris-42): Test more detailed when we will have something in db. - db.db_create() - db.db_drop() - db.db_create() diff --git a/tests/test.py b/tests/test.py deleted file mode 100644 index 9904421..0000000 --- a/tests/test.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2013: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import mock -from oslotest import base - -from logaas import db -from logaas.openstack.common.fixture import config - - -class DatabaseFixture(config.Config): - """Create clean DB before starting test.""" - def setUp(self): - super(DatabaseFixture, self).setUp() - db.db_cleanup() - self.conf.set_default('connection', "sqlite://", group='database') - db.db_drop() - db.db_create() - - -class TestCase(base.BaseTestCase): - """Test case base class for all unit tests.""" - - def setUp(self): - super(TestCase, self).setUp() - self.addCleanup(mock.patch.stopall) - - -class DBTestCase(TestCase): - """Base class for tests which use DB.""" - - def setUp(self): - super(TestCase, self).setUp() - self.useFixture(DatabaseFixture()) diff --git a/tools/config/check_uptodate.sh b/tools/config/check_uptodate.sh deleted file mode 100755 index 9591fe8..0000000 --- a/tools/config/check_uptodate.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -PROJECT_NAME=${PROJECT_NAME:-logaas} -CFGFILE_NAME=${PROJECT_NAME}.conf.sample - -if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then - CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME} -elif [ -e etc/${CFGFILE_NAME} ]; then - CFGFILE=etc/${CFGFILE_NAME} -else - echo "${0##*/}: can not find config file" - exit 1 -fi - -TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX` -trap "rm -rf $TEMPDIR" EXIT - -tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR} - -if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE} -then - echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date." - echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh." - exit 1 -fi diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh deleted file mode 100755 index 760031b..0000000 --- a/tools/config/generate_sample.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash - -# Generate sample configuration for your project. -# -# Aside from the command line flags, it also respects a config file which -# should be named oslo.config.generator.rc and be placed in the same directory. -# -# You can then export the following variables: -# LOGAAS_CONFIG_GENERATOR_EXTRA_MODULES: list of modules to interrogate for options. -# LOGAAS_CONFIG_GENERATOR_EXTRA_LIBRARIES: list of libraries to discover. -# LOGAAS_CONFIG_GENERATOR_EXCLUDED_FILES: list of files to remove from automatic listing. - -print_hint() { - echo "Try \`${0##*/} --help' for more information." >&2 -} - -PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \ - --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@") - -if [ $? != 0 ] ; then print_hint ; exit 1 ; fi - -eval set -- "$PARSED_OPTIONS" - -while true; do - case "$1" in - -h|--help) - echo "${0##*/} [options]" - echo "" - echo "options:" - echo "-h, --help show brief help" - echo "-b, --base-dir=DIR project base directory" - echo "-p, --package-name=NAME project package name" - echo "-o, --output-dir=DIR file output directory" - echo "-m, --module=MOD extra python module to interrogate for options" - echo "-l, --library=LIB extra library that registers options for discovery" - exit 0 - ;; - -b|--base-dir) - shift - BASEDIR=`echo $1 | sed -e 's/\/*$//g'` - shift - ;; - -p|--package-name) - shift - PACKAGENAME=`echo $1` - shift - ;; - -o|--output-dir) - shift - OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'` - shift - ;; - -m|--module) - shift - MODULES="$MODULES -m $1" - shift - ;; - -l|--library) - shift - LIBRARIES="$LIBRARIES -l $1" - shift - ;; - --) - break - ;; - esac -done - -BASEDIR=${BASEDIR:-`pwd`} -if ! [ -d $BASEDIR ] -then - echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1 -elif [[ $BASEDIR != /* ]] -then - BASEDIR=$(cd "$BASEDIR" && pwd) -fi - -PACKAGENAME=${PACKAGENAME:-$(python setup.py --name)} -TARGETDIR=$BASEDIR/$PACKAGENAME -if ! [ -d $TARGETDIR ] -then - echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1 -fi - -OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc} -# NOTE(bnemec): Some projects put their sample config in etc/, -# some in etc/$PACKAGENAME/ -if [ -d $OUTPUTDIR/$PACKAGENAME ] -then - OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME -elif ! [ -d $OUTPUTDIR ] -then - echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2 - exit 1 -fi - -BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'` -find $TARGETDIR -type f -name "*.pyc" -delete -FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \ - -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u) - -RC_FILE="`dirname $0`/oslo.config.generator.rc" -if test -r "$RC_FILE" -then - source "$RC_FILE" -fi - -for filename in ${LOGAAS_CONFIG_GENERATOR_EXCLUDED_FILES}; do - FILES="${FILES[@]/$filename/}" -done - -for mod in ${LOGAAS_CONFIG_GENERATOR_EXTRA_MODULES}; do - MODULES="$MODULES -m $mod" -done - -for lib in ${LOGAAS_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do - LIBRARIES="$LIBRARIES -l $lib" -done - -export EVENTLET_NO_GREENDNS=yes - -OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) -[ "$OS_VARS" ] && eval "unset \$OS_VARS" -DEFAULT_MODULEPATH=logaas.openstack.common.config.generator -MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH} -OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample -python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE - -# Hook to allow projects to append custom config file snippets -CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null) -for CONCAT_FILE in $CONCAT_FILES; do - cat $CONCAT_FILE >> $OUTPUTFILE -done diff --git a/tools/lint.py b/tools/lint.py deleted file mode 100644 index 69b88ca..0000000 --- a/tools/lint.py +++ /dev/null @@ -1,34 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2013 Intel Corporation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# @author: Zhongyue Luo, Intel Corporation. -# - -import sys - -from pylint import lint - - -ENABLED_PYLINT_MSGS = ['W0611'] - - -def main(dirpath): - enable_opt = '--enable=%s' % ','.join(ENABLED_PYLINT_MSGS) - lint.Run(['--reports=n', '--disable=all', enable_opt, dirpath]) - -if __name__ == '__main__': - main(sys.argv[1]) diff --git a/tools/patch_tox_venv.py b/tools/patch_tox_venv.py deleted file mode 100644 index dc9ce83..0000000 --- a/tools/patch_tox_venv.py +++ /dev/null @@ -1,50 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import sys - -import install_venv_common as install_venv # noqa - - -def first_file(file_list): - for candidate in file_list: - if os.path.exists(candidate): - return candidate - - -def main(argv): - root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - - venv = os.environ['VIRTUAL_ENV'] - - pip_requires = first_file([ - os.path.join(root, 'requirements.txt'), - os.path.join(root, 'tools', 'pip-requires'), - ]) - test_requires = first_file([ - os.path.join(root, 'test-requirements.txt'), - os.path.join(root, 'tools', 'test-requires'), - ]) - py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) - project = 'oslo' - install = install_venv.InstallVenv(root, venv, pip_requires, test_requires, - py_version, project) - #NOTE(dprince): For Tox we only run post_process (which patches files, etc) - install.post_process() - -if __name__ == '__main__': - main(sys.argv) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 1efaf73..0000000 --- a/tox.ini +++ /dev/null @@ -1,42 +0,0 @@ -[tox] -minversion = 1.6 -skipsdist = True -envlist = py26,py27,pep8 - -[testenv] -setenv = VIRTUAL_ENV={envdir} - LANG=en_US.UTF-8 - LANGUAGE=en_US:en - LC_ALL=C -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -install_command = pip install -U {opts} {packages} -usedevelop = True -commands = python setup.py testr --slowest --testr-args='{posargs}' -distribute = false - -[testenv:pep8] -commands = flake8 -distribute = false - -[testenv:venv] -commands = {posargs} - -[testenv:cover] -commands = python setup.py testr --coverage --testr-args='{posargs}' - -[testenv:docs] -changedir = doc/source -commands = make html - -[tox:jenkins] -downloadcache = ~/cache/pip - -[flake8] -ignore = E126,H703 -show-source = true -builtins = _ -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,*logaas/openstack* - -[hacking] -import_exceptions = logaas.openstack.common.gettextutils._