Drydock documentation via build_sphinx.

This PS adds tooling and automation to automatically generate
Drydock's documentation into feature-rich HTML pages that can
be hosted.

To run the documentation job, simply execute:

    tox -e docs

A future PS should add warning_is_error to 'build_sphinx' in
setup.py once the import warnings are addressed.

Change-Id: I91a3c585b2c27096e7fde12d180638d1ae4bdb81

.gitignore

@@ -62,6 +62,7 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/*/_static/
 
 # PyBuilder
 target/

docs/source/conf.py

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+#
+# drydock documentation build configuration file, created by
+# sphinx-quickstart on Sat Sep 16 03:40:50 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.todo',
+    'sphinx.ext.viewcode',
+    'oslo_config.sphinxconfiggen',
+    'oslo_policy.sphinxpolicygen'
+]
+
+# oslo_config.sphinxconfiggen options
+config_generator_config_file = '../../etc/drydock/drydock-config-generator.conf'
+sample_config_basename = '_static/drydock'
+
+# oslo_policy.sphinxpolicygen options
+policy_generator_config_file = '../../etc/drydock/drydock-policy-generator.conf'
+sample_policy_basename = '_static/drydock'
+
+# Add any paths that contain templates here, relative to this directory.
+# templates_path = []
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'drydock'
+copyright = u'2017, Drydock Authors'
+author = u'Drydock Authors'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u'0.1.0'
+# The full version, including alpha/beta/rc tags.
+release = u'0.1.0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+import sphinx_rtd_theme
+html_theme = "sphinx_rtd_theme"
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'drydockdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'drydock.tex', u'Drydock Documentation',
+     u'Drydock Authors', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'Drydock', u'Drydock Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'Drydock', u'Drydock Documentation',
+     author, 'Drydock',
+     'An orchestrator to translate a host topology to a provisioned set of hosts.',
+     'Miscellaneous'),
+]

docs/source/drydock_client.rst

@@ -1,3 +1,19 @@
+..
+      Copyright 2017 AT&T Intellectual Property.
+      All Rights Reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
 ===========================================================
 drydock_client - client for drydock_provisioner RESTful API
 ===========================================================

docs/source/getting_started.rst

@@ -1,3 +1,19 @@
+..
+      Copyright 2017 AT&T Intellectual Property.
+      All Rights Reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
 =======================================
 Installing Drydock in a Dev Environment
 =======================================

docs/source/index.rst

@@ -0,0 +1,52 @@
+..
+      Copyright 2017 AT&T Intellectual Property.
+      All Rights Reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+====================================
+Welcome to Drydock's documentation!
+====================================
+
+Drydock is a python REST orchestrator to translate a YAML host topology to a
+provisioned set of hosts and provide a set of cloud-init post-provisioning
+instructions.
+
+User's Guide
+============
+
+Drydock Configuration Guide
+---------------------------
+
+.. toctree::
+   :maxdepth: 2
+
+   getting_started
+   sampleconf
+   policy-enforcement
+
+Client Documentation
+--------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   drydock_client
+
+Topology Documentation
+----------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   topology

docs/source/policy-enforcement.rst

@@ -0,0 +1,25 @@
+..
+    Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+Sample Policy File
+==================
+The following is a sample Drydock policy file for adaptation and use. It is
+auto-generated from Drydock when this documentation is built, so
+if you are having issues with an option, please compare your version of
+Drydock with the version of this documentation.
+
+The sample policy file can also be viewed in `file form <_static/drydock.policy.yaml.sample>`_.
+
+.. literalinclude:: _static/drydock.policy.yaml.sample

docs/source/sampleconf.rst

@@ -0,0 +1,26 @@
+..
+    Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+Sample Configuration File
+==========================
+
+The following is a sample Drydock configuration for adaptation and use. It is
+auto-generated from Drydock when this documentation is built, so
+if you are having issues with an option, please compare your version of
+Patrole with the version of this documentation.
+
+The sample configuration can also be viewed in `file form <_static/drydock.conf.sample>`_.
+
+.. literalinclude:: _static/drydock.conf.sample

docs/source/topology.rst

@@ -1,3 +1,19 @@
+..
+      Copyright 2017 AT&T Intellectual Property.
+      All Rights Reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
 =======================
 Authoring Site Topology
 =======================

etc/drydock/drydock.conf.sample

@@ -0,0 +1,333 @@
+[DEFAULT]
+
+#
+# From drydock_provisioner
+#
+
+# Polling interval in seconds for checking subtask or downstream status (integer
+# value)
+#poll_interval = 10
+
+
+[bootdata]
+
+#
+# From drydock_provisioner
+#
+
+# Path to file to distribute for prom_init.sh (string value)
+#prom_init = /etc/drydock/bootdata/join.sh
+
+
+[keystone_authtoken]
+
+#
+# From drydock_provisioner
+#
+
+# Authentication URL (string value)
+#auth_url = <None>
+
+# Domain ID to scope to (string value)
+#domain_id = <None>
+
+# Domain name to scope to (string value)
+#domain_name = <None>
+
+# Project ID to scope to (string value)
+# Deprecated group/name - [keystone_authtoken]/tenant-id
+#project_id = <None>
+
+# Project name to scope to (string value)
+# Deprecated group/name - [keystone_authtoken]/tenant-name
+#project_name = <None>
+
+# Domain ID containing project (string value)
+#project_domain_id = <None>
+
+# Domain name containing project (string value)
+#project_domain_name = <None>
+
+# Trust ID (string value)
+#trust_id = <None>
+
+# Optional domain ID to use with v3 and v2 parameters. It will be used for both
+# the user and project domain in v3 and ignored in v2 authentication. (string
+# value)
+#default_domain_id = <None>
+
+# Optional domain name to use with v3 API and v2 parameters. It will be used for
+# both the user and project domain in v3 and ignored in v2 authentication.
+# (string value)
+#default_domain_name = <None>
+
+# User id (string value)
+#user_id = <None>
+
+# Username (string value)
+# Deprecated group/name - [keystone_authtoken]/user-name
+#username = <None>
+
+# User's domain id (string value)
+#user_domain_id = <None>
+
+# User's domain name (string value)
+#user_domain_name = <None>
+
+# User's password (string value)
+#password = <None>
+
+#
+# From keystonemiddleware.auth_token
+#
+
+# Complete "public" Identity API endpoint. This endpoint should not be an
+# "admin" endpoint, as it should be accessible by all end users. Unauthenticated
+# clients are redirected to this endpoint to authenticate. Although this
+# endpoint should  ideally be unversioned, client support in the wild varies.
+# If you're using a versioned v2 endpoint here, then this  should *not* be the
+# same endpoint the service user utilizes  for validating tokens, because normal
+# end users may not be  able to reach that endpoint. (string value)
+#auth_uri = <None>
+
+# API version of the admin Identity API endpoint. (string value)
+#auth_version = <None>
+
+# Do not handle authorization requests within the middleware, but delegate the
+# authorization decision to downstream WSGI components. (boolean value)
+#delay_auth_decision = false
+
+# Request timeout value for communicating with Identity API server. (integer
+# value)
+#http_connect_timeout = <None>
+
+# How many times are we trying to reconnect when communicating with Identity API
+# Server. (integer value)
+#http_request_max_retries = 3
+
+# Request environment key where the Swift cache object is stored. When
+# auth_token middleware is deployed with a Swift cache, use this option to have
+# the middleware share a caching backend with swift. Otherwise, use the
+# ``memcached_servers`` option instead. (string value)
+#cache = <None>
+
+# Required if identity server requires client certificate (string value)
+#certfile = <None>
+
+# Required if identity server requires client certificate (string value)
+#keyfile = <None>
+
+# A PEM encoded Certificate Authority to use when verifying HTTPs connections.
+# Defaults to system CAs. (string value)
+#cafile = <None>
+
+# Verify HTTPS connections. (boolean value)
+#insecure = false
+
+# The region in which the identity server can be found. (string value)
+#region_name = <None>
+
+# Directory used to cache files related to PKI tokens. (string value)
+#signing_dir = <None>
+
+# Optionally specify a list of memcached server(s) to use for caching. If left
+# undefined, tokens will instead be cached in-process. (list value)
+# Deprecated group/name - [keystone_authtoken]/memcache_servers
+#memcached_servers = <None>
+
+# In order to prevent excessive effort spent validating tokens, the middleware
+# caches previously-seen tokens for a configurable duration (in seconds). Set to
+# -1 to disable caching completely. (integer value)
+#token_cache_time = 300
+
+# Determines the frequency at which the list of revoked tokens is retrieved from
+# the Identity service (in seconds). A high number of revocation events combined
+# with a low cache duration may significantly reduce performance. Only valid for
+# PKI tokens. (integer value)
+#revocation_cache_time = 10
+
+# (Optional) If defined, indicate whether token data should be authenticated or
+# authenticated and encrypted. If MAC, token data is authenticated (with HMAC)
+# in the cache. If ENCRYPT, token data is encrypted and authenticated in the
+# cache. If the value is not one of these options or empty, auth_token will
+# raise an exception on initialization. (string value)
+# Allowed values: None, MAC, ENCRYPT
+#memcache_security_strategy = None
+
+# (Optional, mandatory if memcache_security_strategy is defined) This string is
+# used for key derivation. (string value)
+#memcache_secret_key = <None>
+
+# (Optional) Number of seconds memcached server is considered dead before it is
+# tried again. (integer value)
+#memcache_pool_dead_retry = 300
+
+# (Optional) Maximum total number of open connections to every memcached server.
+# (integer value)
+#memcache_pool_maxsize = 10
+
+# (Optional) Socket timeout in seconds for communicating with a memcached
+# server. (integer value)
+#memcache_pool_socket_timeout = 3
+
+# (Optional) Number of seconds a connection to memcached is held unused in the
+# pool before it is closed. (integer value)
+#memcache_pool_unused_timeout = 60
+
+# (Optional) Number of seconds that an operation will wait to get a memcached
+# client connection from the pool. (integer value)
+#memcache_pool_conn_get_timeout = 10
+
+# (Optional) Use the advanced (eventlet safe) memcached client pool. The
+# advanced pool will only work under python 2.x. (boolean value)
+#memcache_use_advanced_pool = false
+
+# (Optional) Indicate whether to set the X-Service-Catalog header. If False,
+# middleware will not ask for service catalog on token validation and will not
+# set the X-Service-Catalog header. (boolean value)
+#include_service_catalog = true
+
+# Used to control the use and type of token binding. Can be set to: "disabled"
+# to not check token binding. "permissive" (default) to validate binding
+# information if the bind type is of a form known to the server and ignore it if
+# not. "strict" like "permissive" but if the bind type is unknown the token will
+# be rejected. "required" any form of token binding is needed to be allowed.
+# Finally the name of a binding method that must be present in tokens. (string
+# value)
+#enforce_token_bind = permissive
+
+# If true, the revocation list will be checked for cached tokens. This requires
+# that PKI tokens are configured on the identity server. (boolean value)
+#check_revocations_for_cached = false
+
+# Hash algorithms to use for hashing PKI tokens. This may be a single algorithm
+# or multiple. The algorithms are those supported by Python standard
+# hashlib.new(). The hashes will be tried in the order given, so put the
+# preferred one first for performance. The result of the first hash will be
+# stored in the cache. This will typically be set to multiple values only while
+# migrating from a less secure algorithm to a more secure one. Once all the old
+# tokens are expired this option should be set to a single value for better
+# performance. (list value)
+#hash_algorithms = md5
+
+# Authentication type to load (string value)
+# Deprecated group/name - [keystone_authtoken]/auth_plugin
+#auth_type = <None>
+
+# Config Section from which to load plugin specific options (string value)
+#auth_section = <None>
+
+
+[logging]
+
+#
+# From drydock_provisioner
+#
+
+# Global log level for Drydock (string value)
+#log_level = INFO
+
+# Logger name for the top-level logger (string value)
+#global_logger_name = drydock
+
+# Logger name for OOB driver logging (string value)
+#oobdriver_logger_name = ${global_logger_name}.oobdriver
+
+# Logger name for Node driver logging (string value)
+#nodedriver_logger_name = ${global_logger_name}.nodedriver
+
+# Logger name for API server logging (string value)
+#control_logger_name = ${global_logger_name}.control
+
+
+[maasdriver]
+
+#
+# From drydock_provisioner
+#
+
+# The API key for accessing MaaS (string value)
+#maas_api_key = <None>
+
+# The URL for accessing MaaS API (string value)
+#maas_api_url = <None>
+
+# Polling interval for querying MaaS status in seconds (integer value)
+#poll_interval = 10
+
+
+[oslo_policy]
+
+#
+# From oslo.policy
+#
+
+# The file that defines policies. (string value)
+# Deprecated group/name - [DEFAULT]/policy_file
+#policy_file = policy.json
+
+# Default rule. Enforced when a requested rule is not found. (string value)
+# Deprecated group/name - [DEFAULT]/policy_default_rule
+#policy_default_rule = default
+
+# Directories where policy configuration files are stored. They can be relative
+# to any directory in the search path defined by the config_dir option, or
+# absolute paths. The file defined by policy_file must exist for these
+# directories to be searched.  Missing or empty directories are ignored. (multi
+# valued)
+# Deprecated group/name - [DEFAULT]/policy_dirs
+#policy_dirs = policy.d
+
+
+[plugins]
+
+#
+# From drydock_provisioner
+#
+
+# Module path string of a input ingester to enable (multi valued)
+#ingester = drydock_provisioner.ingester.plugins.yaml.YamlIngester
+
+# Module path string of a OOB driver to enable (multi valued)
+#oob_driver = drydock_provisioner.drivers.oob.pyghmi_driver.PyghmiDriver
+
+# Module path string of the Node driver to enable (string value)
+#node_driver = drydock_provisioner.drivers.node.maasdriver.driver.MaasNodeDriver
+
+# Module path string of the Network driver enable (string value)
+#network_driver = <None>
+
+
+[timeouts]
+
+#
+# From drydock_provisioner
+#
+
+# Fallback timeout when a specific one is not configured (integer value)
+#drydock_timeout = 5
+
+# Timeout in minutes for creating site network templates (integer value)
+#create_network_template = 2
+
+# Timeout in minutes for creating user credentials (integer value)
+#configure_user_credentials = 2
+
+# Timeout in minutes for initial node identification (integer value)
+#identify_node = 10
+
+# Timeout in minutes for node commissioning and hardware configuration (integer
+# value)
+#configure_hardware = 30
+
+# Timeout in minutes for configuring node networking (integer value)
+#apply_node_networking = 5
+
+# Timeout in minutes for configuring node storage (integer value)
+#apply_node_storage = 5
+
+# Timeout in minutes for configuring node platform (integer value)
+#apply_node_platform = 5
+
+# Timeout in minutes for deploying a node (integer value)
+#deploy_node = 45

requirements-test.txt

@@ -8,3 +8,5 @@ oslo.config[fixtures]
 yapf
 flake8
 bandit>=1.1.0
+sphinx>=1.6.2
+sphinx_rtd_theme==0.2.4

setup.py

@@ -17,6 +17,10 @@
 # scripts
 
 from setuptools import setup
+from sphinx.setup_command import BuildDoc
+
+cmdclass = {'build_sphinx': BuildDoc}
+
 
 setup(
     name='drydock_provisioner',
@@ -48,4 +52,12 @@ setup(
         'drydock_provisioner = drydock_provisioner.policy:list_policies',
         'console_scripts':
         'drydock = drydock_provisioner.cli.commands:drydock'
+    },
+    cmdclass=cmdclass,
+    command_options={
+        'build_sphinx': {
+            'source_dir': ('setup.py', 'docs/source'),
+            'build_dir': ('setup.py', 'docs/build'),
+            'all_files': ('setup.py', 1),
+        }
     })

tox.ini

@@ -39,3 +39,9 @@ commands = bandit -r drydock_provisioner -n 5
 ignore=E302,H306,H304,D101,D102,D103,D104
 exclude= venv,.venv,.git,.idea,.tox,*.egg-info,*.eggs,bin,dist,./build/
 max-line-length=119
+
+[testenv:docs]
+whitelist_externals=rm
+commands =
+    rm -rf docs/build
+    python setup.py build_sphinx {posargs}