From 34454593ffc5f4a96f3f8228fde07c4c0e5aee74 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 1 May 2017 12:12:30 -0400 Subject: [PATCH] add documentation Change-Id: I37a11f198d020067283ca5c154e610f2ff6770c1 Signed-off-by: Doug Hellmann --- README.rst | 6 +- demo/tiny-resources.yml | 3 +- doc/source/background.rst | 57 +++++++++++++++++ doc/source/conf.py | 4 +- doc/source/contributing.rst | 7 ++- doc/source/glossary.rst | 10 +++ doc/source/index.rst | 32 +++++----- doc/source/installation.rst | 29 ++++++--- doc/source/readme.rst | 1 - doc/source/resource_file.rst | 83 ++++++++++++++++++++++++ doc/source/usage.rst | 119 +++++++++++++++++++++++++++++++++-- setup.cfg | 1 + 12 files changed, 314 insertions(+), 38 deletions(-) create mode 100644 doc/source/background.rst create mode 100644 doc/source/glossary.rst delete mode 100644 doc/source/readme.rst create mode 100644 doc/source/resource_file.rst diff --git a/README.rst b/README.rst index ab14279..d8c681e 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ -================================================== - downpour -- OpenStack Tenant Data Migration Tool -================================================== +=================================================== + downpour --- OpenStack Tenant Data Migration Tool +=================================================== downpour exports tenant data from an OpenStack cloud to create a set of Ansible playbooks for importing the data into another cloud. diff --git a/demo/tiny-resources.yml b/demo/tiny-resources.yml index 56f5fd6..efd0990 100644 --- a/demo/tiny-resources.yml +++ b/demo/tiny-resources.yml @@ -1,7 +1,8 @@ # Resource file for downpour using the instance created in tiny.yml. servers: - name: downpour-demo-tiny - # Create the server using a separate key than it was created with in tiny.yml. + # Create the server using a separate key than + # it was created with in tiny.yml. key_name: downpour-demo2 keypairs: - name: downpour-demo diff --git a/doc/source/background.rst b/doc/source/background.rst new file mode 100644 index 0000000..2c650ff --- /dev/null +++ b/doc/source/background.rst @@ -0,0 +1,57 @@ +============ + Background +============ + +Downpour is being created to solve the problem of moving workloads +between clouds. It is only one of several possible approaches to the +problem, and fits into a very specific niche at the hard end of the +range of use cases. + +.. list-table:: + :header-rows: 1 + + - * + * Easy + * Moderate + * Hard + - * **Ownership** + * Operator + * Admin + * Tenant + - * **Backend** + * Shared storage + * Fast interconnect + * Shared nothing + - * **Applications** + * One per tenant + * Multi-app with naming conventions + * Rats nest + +Downpour does not assume the user has an special access to the cloud, +either as an operator with access to backend systems or via admin +APIs. + +Downpour does not assume that the source and destination clouds are +connected in any way. Not only is it possible to move data between +clouds that do not share backend services, it is possible to move data +between clouds that cannot be accessed from the same system at the +same time. + +Downpour does not make any assumptions about the mapping between +applications and tenants. It is possible to extract only part of the +resources owned by a tenant. The grouping is completely up to the +user, and can represent an application or a single node in a multi-VM +configuration. + +Downpour does not assume the source and destination clouds are build +using the same architecture or configured in the same way. As long as +the two clouds pass the standard OpenStack interoperability tests, it +should be possible to use Downpour to move your workload. + +These requirements do come with trade-offs, the impact of which will +depend on how "cloud native" an application really is. For example, +the benefits of copy-on-write images may be lost during the migration +if the entire image from each VM needs to be uploaded into the new +cloud. The UUIDs associated with resources will also change, since +there is no way to guarantee the assignment of a specific UUID for +resources created in a separate cloud. diff --git a/doc/source/conf.py b/doc/source/conf.py index e9f6af3..f6251c3 100755 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,7 +23,6 @@ sys.path.insert(0, os.path.abspath('../..')) extensions = [ 'sphinx.ext.autodoc', #'sphinx.ext.intersphinx', - 'oslosphinx' ] # autodoc generation is a bit aggressive and a nuisance when doing heavy @@ -38,7 +37,7 @@ master_doc = 'index' # General information about the project. project = u'downpour' -copyright = u'2016, OpenStack Foundation' +copyright = u'2017, Downpour contributors' # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = True @@ -57,6 +56,7 @@ pygments_style = 'sphinx' # html_theme_path = ["."] # html_theme = '_theme' # html_static_path = ['static'] +html_theme = 'nature' # Output file base name for HTML help builder. htmlhelp_basename = '%sdoc' % project diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 1728a61..2ca75d1 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -1,4 +1,5 @@ -============ -Contributing -============ +============== + Contributing +============== + .. include:: ../../CONTRIBUTING.rst diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst new file mode 100644 index 0000000..72cc884 --- /dev/null +++ b/doc/source/glossary.rst @@ -0,0 +1,10 @@ +========== + Glossary +========== + +.. glossary:: + + resource file + A YAML file containing explicitly identified resources to be + exported. See :doc:`resource_file` for more details. + diff --git a/doc/source/index.rst b/doc/source/index.rst index 253b4e8..4db736a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,25 +1,25 @@ -.. downpour documentation master file, created by - sphinx-quickstart on Tue Jul 9 22:26:36 2013. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +================================================== + downpour -- OpenStack Tenant Data Migration Tool +================================================== -Welcome to downpour's documentation! -======================================================== +downpour exports tenant data from an OpenStack cloud to create a set +of Ansible_ playbooks for importing the data into another cloud. -Contents: +.. note:: + + The project is in a very very early prototyping stage. + +.. _ansible: https://www.ansible.com + +Contents +======== .. toctree:: :maxdepth: 2 - readme + background installation usage + resource_file contributing - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - + glossary diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 68fecc7..f5d23f1 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -1,12 +1,27 @@ -============ -Installation -============ +============== + Installation +============== + +Prerequisites +============= + +Downpour is written to take advantage of features of Python 3.5, so +you will need a Python 3.5+ interpreter installed. + +Installing with pip +=================== At the command line:: - $ pip install downpour + $ pip install os-downpour -Or, if you have virtualenvwrapper installed:: +.. note:: The dist name for downpour is ``os-downpour``. - $ mkvirtualenv downpour - $ pip install downpour +Cloud Access Credentials +======================== + +downpour uses `os-client-config`_ for settings related to accessing +the cloud. Fill in your ``clouds.yaml`` or use the environment +variables or command line arguments provided. + +.. _os-client-config: http://docs.openstack.org/developer/os-client-config/ diff --git a/doc/source/readme.rst b/doc/source/readme.rst deleted file mode 100644 index a6210d3..0000000 --- a/doc/source/readme.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../README.rst diff --git a/doc/source/resource_file.rst b/doc/source/resource_file.rst new file mode 100644 index 0000000..ee0275d --- /dev/null +++ b/doc/source/resource_file.rst @@ -0,0 +1,83 @@ +====================== + Resource File Format +====================== + +A Downpour resource file is a YAML file containing explicitly +identified resources to be exported, along with instructions for how +to handle the export. + +``keypairs`` +============ + +The keypairs section lists the names of the keypairs to be +exported. Keys associated with servers are exported automatically, but +if it is important to move keys not in use by any of the servers those +keys can be listed separately. + +Each item in the keypairs list should be a mapping with a value for +``name``. + +:: + + keypairs: + - name: downpour-demo + +``images`` +========== + +The images section lists the names of the images to be exported. + +Each item in the images list should be a mapping with a value for +``name``. + +:: + + images: + - name: cirros-0.3.5-x86_64-disk + +``volumes`` +=========== + +The volumes section lists the names and settings for the unattached +volumes to be exported. This section should **not** include volumes +attached to servers, because those are exported as part of exporting +the server definition. + +Each item in the images list should be a mapping with a value for +``name`` and an optional boolean value for ``save_state``, indicating +whether the contents of the volume should be exported. If +``save_state`` is false, a new volume with the same name and size will +be created but it will be empty. The default is to save the contents +of the volume. + +:: + + volumes: + - name: downpour-demo-unattached + save_state: false + +``servers`` +=========== + +The servers section lists the names and settings for the virtual +machines to be exported. + +Each item in the images list should be a mapping with a value for +``name``. It can also contain an optional boolean value for +``save_state``, indicating whether the contents of the VM should be +exported. If ``save_state`` is false, a new VM with the same name and +flavor will be created, but it will not contain any of the files from +the current VM. The default is to save the contents of the volume. + +If an optional ``key_name`` setting is given, the new VM will be +initialized using that ssh keypair instead of the one already +associated with the server. The keypair does not need to exist on the +source system. + +:: + + servers: + - name: downpour-demo-tiny + # Create the server using a separate key than + # it was created with in tiny.yml. + key_name: downpour-demo2 diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 5946e4b..ac9a5d7 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -1,7 +1,116 @@ -======== -Usage -======== +======= + Usage +======= -To use downpour in a project:: +Downpour uses a four step process. Between each step it is possible to +stop and modify the data that has been prepared to pass to the next +step. - import downpour +1. Identify Resources to Export +=============================== + +The first phase of using Downpour is to identify exactly what +resources will be exported from the cloud to build the :term:`resource +file`. This step can be performed by hand by creating the required +input file in a text editor, or the file can be build using the +``query`` command. + +The resource file is a YAML file with sections for the principle +resource types, ``keypairs``, ``images``, ``volumes``, and +``servers``. Resources are identified by name, and may also include +extra parameters to control how the export and re-import operations +are performed. For example, this resource file causes the +``downpour-demo-tiny`` server to be exported but when it is recreated +a different ssh key is used to provide access to log in. + +.. literalinclude:: ../../demo/tiny-resources.yml + +The ``downpour query`` command also can be used to find resources +visible in the cloud, and add them to the resource file. It supports +wildcard patterns in names and other criteria for filtering +resources. For example, this command finds all servers with "``tiny``" +in their name. + +:: + + $ downpour query --server-name '*tiny*' export.yml + +.. seealso:: + + :doc:`resource_file` includes more details about resource files. + +2. Exporting Resources +====================== + +The second phase of operation is to actually export the resources from +the cloud using ``downpour export``, passing the resource file as +input. Downpour starts by processing the resources listed in the file +explicitly, and identifies any extra dependencies needed to recreate +the configuration of those resources. For example, the networks, +subnets, and security groups used by a server are exported +automatically, as are the volumes attached to the server. + +:: + + $ downpour export export.yml ./export/ + +The output for the export process is an Ansible_ playbook to recreate +the resources, with all relationships intact. For images, volumes, and +servers with the ``save-state`` flag set to true, the content of the +resource will be downloaded and saved to the output directory where it +can be used to recreate the resource. + +3. Importing Resources +====================== + +The import phase uses ``ansible-playbook`` to run the playbook created +by the exporter. + +.. note:: + + Although Downpour currently requires Python 3.5 or greater, Ansible + is a Python 2.x application. If you are using ``pip`` and + ``virtualenv`` to install the tools, you will need to install them + in separate virtual environments. + +Ansible uses uses `os-client-config`_ for settings related to +accessing the cloud. The simplest way to configure the cloud is via a +``clouds.yaml`` file, but any mechanism supported by Ansible will +work. The credentials used for the import phase do not need to be the +same as the export phase. In fact, they're likely to be completely +different because they will refer to a separate cloud's API endpoints. + +Downpour supports some customizations during export, such as changing +the ssh key to be used for accessing a server. Other changes can be +made by editing the playbook before running it. + +The playbook produced by Downpour creates each resource, then adds a +line to a file ``uuids.csv`` to map the UUID in the source cloud to +the UUID in the target cloud. This file may be useful for updating +scripts or other configuration that rely on the UUID instead of a +unique name for the resource. + +:: + + "Resource Type","Resource Name","Old","New" + "security group","downpour-demo","6deea469-54bd-4846-b12a-79fa6b482280","a4b80ffc-bc51-485c-915a-9ba9a7b4dcf0" + "volume","downpour-demo-tiny","256868c6-441f-4cd3-96fd-bda92c33822c","62e5616c-9a8c-44e2-bd14-4685b905ea94" + "security group","downpour-demo","3c7dcb77-d9ac-4af1-ba95-3f5d89a85227","a4b80ffc-bc51-485c-915a-9ba9a7b4dcf0" + "volume","downpour-demo-tiny","a6192546-c36e-4bee-ad00-8229e0b0efc5","62e5616c-9a8c-44e2-bd14-4685b905ea94" + "network","private","56a86bdb-13b2-4c9f-b8f5-a942d52602b5","f3027502-e4a2-4610-81fb-c6df99ead5c3" + "subnet","ipv6-private-subnet","8d736fe4-6b8f-4bf5-a38e-b511dce21f7f","01025e33-703b-4aa4-b6ec-80036bb3679b" + "subnet","private-subnet","e6baf9f4-09b5-4292-8236-3cca609ec2a3","2f9a1686-8125-4316-acd3-dbee51c44c1d" + "keypair","downpour-demo","downpour-demo","downpour-demo" + "image","cirros-0.3.5-x86_64-disk","570ec7bd-011b-4fbe-9968-626225654a7f","570ec7bd-011b-4fbe-9968-626225654a7f" + +.. _ansible: https://www.ansible.com +.. _os-client-config: http://docs.openstack.org/developer/os-client-config/ + +4. Decomissioning Resources +=========================== + +Downpour is not a live-migration tool, and it does not delete any +resources from the source cloud. This allows you to perform +application-specific migration (such as a final database sync) before +updating any load balancers or DNS records and deleting old +information. diff --git a/setup.cfg b/setup.cfg index 1c5436e..26d301e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ console_scripts = source-dir = doc/source build-dir = doc/build all_files = 1 +warning-is-error = 1 [upload_sphinx] upload-dir = doc/build/html