From 4b3adfeea6f2e8470fe99543a45777e5ec27dc65 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Fri, 22 Mar 2019 15:26:41 -0700 Subject: [PATCH] Organize documentation by subject area This splits all of the current job and role documentation into files by subject area so that jobs and roles are easier for users to find. This will require that any future new jobs or roles add a line to the appropriate area of the documentation, since that can no longer be done automatically. A linter check is added to ensure that every job and role continue to be documented. After this refactor, we can begin to enhance the documentation pages so that they include narrative documentation and subsections. Change-Id: Ia6f0e89b57e3cb0d7d1745206384c946506d7ea0 --- doc/source/afs-roles.rst | 7 ++ doc/source/container-roles.rst | 15 ++++ doc/source/deprecated-roles.rst | 5 ++ doc/source/docker-jobs.rst | 6 ++ doc/source/galaxy-roles.rst | 4 + doc/source/general-jobs.rst | 7 ++ doc/source/general-roles.rst | 36 ++++++++ doc/source/index.rst | 2 +- doc/source/jobs.rst | 9 +- doc/source/js-jobs.rst | 9 ++ doc/source/js-roles.rst | 13 +++ doc/source/launchpad-roles.rst | 6 ++ doc/source/log-roles.rst | 13 +++ doc/source/puppet-roles.rst | 6 ++ doc/source/python-jobs.rst | 18 ++++ doc/source/python-roles.rst | 21 +++++ doc/source/roles.rst | 15 +++- doc/source/translation-roles.rst | 4 + tools/check_jobs_documented.py | 144 +++++++++++++++++++++++++++++++ tox.ini | 1 + 20 files changed, 337 insertions(+), 4 deletions(-) create mode 100644 doc/source/afs-roles.rst create mode 100644 doc/source/container-roles.rst create mode 100644 doc/source/deprecated-roles.rst create mode 100644 doc/source/docker-jobs.rst create mode 100644 doc/source/galaxy-roles.rst create mode 100644 doc/source/general-jobs.rst create mode 100644 doc/source/general-roles.rst create mode 100644 doc/source/js-jobs.rst create mode 100644 doc/source/js-roles.rst create mode 100644 doc/source/launchpad-roles.rst create mode 100644 doc/source/log-roles.rst create mode 100644 doc/source/puppet-roles.rst create mode 100644 doc/source/python-jobs.rst create mode 100644 doc/source/python-roles.rst create mode 100644 doc/source/translation-roles.rst create mode 100755 tools/check_jobs_documented.py diff --git a/doc/source/afs-roles.rst b/doc/source/afs-roles.rst new file mode 100644 index 000000000..c2e15e64d --- /dev/null +++ b/doc/source/afs-roles.rst @@ -0,0 +1,7 @@ +AFS Roles +========= + +.. zuul:autorole:: destroy-afs-token +.. zuul:autorole:: upload-afs +.. zuul:autorole:: create-afs-token +.. zuul:autorole:: release-afs-volume diff --git a/doc/source/container-roles.rst b/doc/source/container-roles.rst new file mode 100644 index 000000000..2e2feb360 --- /dev/null +++ b/doc/source/container-roles.rst @@ -0,0 +1,15 @@ +Container Roles +=============== + +.. zuul:autorole:: build-docker-image +.. zuul:autorole:: deploy-openshift +.. zuul:autorole:: install-docker +.. zuul:autorole:: install-kubernetes +.. zuul:autorole:: install-openshift +.. zuul:autorole:: promote-docker-image +.. zuul:autorole:: pull-from-intermediate-registry +.. zuul:autorole:: push-to-intermediate-registry +.. zuul:autorole:: run-buildset-registry +.. zuul:autorole:: upload-docker-image +.. zuul:autorole:: use-buildset-registry +.. zuul:autorole:: use-docker-mirror diff --git a/doc/source/deprecated-roles.rst b/doc/source/deprecated-roles.rst new file mode 100644 index 000000000..89fc76a39 --- /dev/null +++ b/doc/source/deprecated-roles.rst @@ -0,0 +1,5 @@ +Deprecrated and Test Roles +========================== + +.. zuul:autorole:: fetch-zuul-cloner +.. zuul:autorole:: test-mirror-workspace-git-repos diff --git a/doc/source/docker-jobs.rst b/doc/source/docker-jobs.rst new file mode 100644 index 000000000..d77d97b52 --- /dev/null +++ b/doc/source/docker-jobs.rst @@ -0,0 +1,6 @@ +Docker Jobs +=========== + +.. zuul:autojob:: build-docker-image +.. zuul:autojob:: upload-docker-image +.. zuul:autojob:: promote-docker-image diff --git a/doc/source/galaxy-roles.rst b/doc/source/galaxy-roles.rst new file mode 100644 index 000000000..e7f185157 --- /dev/null +++ b/doc/source/galaxy-roles.rst @@ -0,0 +1,4 @@ +Ansible Galaxy Roles +==================== + +.. zuul:autorole:: ansible-galaxy-import diff --git a/doc/source/general-jobs.rst b/doc/source/general-jobs.rst new file mode 100644 index 000000000..314e093bd --- /dev/null +++ b/doc/source/general-jobs.rst @@ -0,0 +1,7 @@ +General Purpose Jobs +==================== + +.. zuul:autojob:: dco-license +.. zuul:autojob:: unittests +.. zuul:autojob:: multinode +.. zuul:autojob:: run-test-command diff --git a/doc/source/general-roles.rst b/doc/source/general-roles.rst new file mode 100644 index 000000000..7a18505a8 --- /dev/null +++ b/doc/source/general-roles.rst @@ -0,0 +1,36 @@ +General Purpose Roles +===================== + +.. zuul:autorole:: add-authorized-keys +.. zuul:autorole:: add-build-sshkey +.. zuul:autorole:: add-gpgkey +.. zuul:autorole:: add-sshkey +.. zuul:autorole:: bindep +.. zuul:autorole:: buildset-artifacts-location +.. zuul:autorole:: configure-mirrors +.. zuul:autorole:: copy-build-sshkey +.. zuul:autorole:: download-artifact +.. zuul:autorole:: emit-job-header +.. zuul:autorole:: git-prepare-nodecache +.. zuul:autorole:: log-inventory +.. zuul:autorole:: mirror-workspace-git-repos +.. zuul:autorole:: multi-node-bridge +.. zuul:autorole:: multi-node-firewall +.. zuul:autorole:: multi-node-hosts-file +.. zuul:autorole:: multi-node-known-hosts +.. zuul:autorole:: persistent-firewall +.. zuul:autorole:: prepare-workspace +.. zuul:autorole:: prepare-workspace-git +.. zuul:autorole:: remove-build-sshkey +.. zuul:autorole:: remove-gpgkey +.. zuul:autorole:: remove-sshkey +.. zuul:autorole:: revoke-sudo +.. zuul:autorole:: sign-artifacts +.. zuul:autorole:: stage-output +.. zuul:autorole:: start-zuul-console +.. zuul:autorole:: test-setup +.. zuul:autorole:: trigger-readthedocs +.. zuul:autorole:: validate-dco-license +.. zuul:autorole:: validate-host +.. zuul:autorole:: version-from-git +.. zuul:autorole:: write-inventory diff --git a/doc/source/index.rst b/doc/source/index.rst index 31017c326..a802f395f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,7 +1,7 @@ .. include:: ../../README.rst .. toctree:: - :maxdepth: 1 + :maxdepth: 2 install policy diff --git a/doc/source/jobs.rst b/doc/source/jobs.rst index 1d722cb91..b1141eb86 100644 --- a/doc/source/jobs.rst +++ b/doc/source/jobs.rst @@ -1,5 +1,10 @@ Jobs -===== +==== -.. zuul:autojobs:: +.. toctree:: + :maxdepth: 1 + general-jobs + python-jobs + js-jobs + docker-jobs diff --git a/doc/source/js-jobs.rst b/doc/source/js-jobs.rst new file mode 100644 index 000000000..200648466 --- /dev/null +++ b/doc/source/js-jobs.rst @@ -0,0 +1,9 @@ +Javascript Jobs +=============== + +.. zuul:autojob:: build-javascript-tarball +.. zuul:autojob:: build-javascript-content +.. zuul:autojob:: nodejs-npm +.. zuul:autojob:: nodejs-npm-run-test +.. zuul:autojob:: nodejs-npm-run-lint +.. zuul:autojob:: nodejs-npm-run-docs diff --git a/doc/source/js-roles.rst b/doc/source/js-roles.rst new file mode 100644 index 000000000..0c628d84d --- /dev/null +++ b/doc/source/js-roles.rst @@ -0,0 +1,13 @@ +Javascript Roles +================ + +.. zuul:autorole:: fetch-javascript-content-tarball +.. zuul:autorole:: fetch-javascript-output +.. zuul:autorole:: fetch-javascript-tarball +.. zuul:autorole:: install-javascript-packages +.. zuul:autorole:: install-nodejs +.. zuul:autorole:: install-yarn +.. zuul:autorole:: nodejs-test-dependencies +.. zuul:autorole:: npm +.. zuul:autorole:: upload-npm +.. zuul:autorole:: yarn diff --git a/doc/source/launchpad-roles.rst b/doc/source/launchpad-roles.rst new file mode 100644 index 000000000..051d3cd1f --- /dev/null +++ b/doc/source/launchpad-roles.rst @@ -0,0 +1,6 @@ +Launchpad Roles +=============== + +.. zuul:autorole:: add-launchpad-credentials +.. zuul:autorole:: remove-launchpad-credentials + diff --git a/doc/source/log-roles.rst b/doc/source/log-roles.rst new file mode 100644 index 000000000..395f2426b --- /dev/null +++ b/doc/source/log-roles.rst @@ -0,0 +1,13 @@ +Log Roles +========= + +.. zuul:autorole:: add-fileserver +.. zuul:autorole:: ara-report +.. zuul:autorole:: ensure-output-dirs +.. zuul:autorole:: fetch-output +.. zuul:autorole:: htmlify-logs +.. zuul:autorole:: merge-output-to-logs +.. zuul:autorole:: publish-artifacts-to-fileserver +.. zuul:autorole:: set-zuul-log-path-fact +.. zuul:autorole:: upload-logs +.. zuul:autorole:: upload-logs-swift diff --git a/doc/source/puppet-roles.rst b/doc/source/puppet-roles.rst new file mode 100644 index 000000000..7ac9e64e0 --- /dev/null +++ b/doc/source/puppet-roles.rst @@ -0,0 +1,6 @@ +Puppet Roles +============ + +.. zuul:autorole:: build-puppet-module +.. zuul:autorole:: fetch-puppet-module-output +.. zuul:autorole:: upload-forge diff --git a/doc/source/python-jobs.rst b/doc/source/python-jobs.rst new file mode 100644 index 000000000..2f39cbd20 --- /dev/null +++ b/doc/source/python-jobs.rst @@ -0,0 +1,18 @@ +Python Jobs +=========== + +.. zuul:autojob:: tox +.. zuul:autojob:: tox-py27 +.. zuul:autojob:: tox-py34 +.. zuul:autojob:: tox-py35 +.. zuul:autojob:: tox-py36 +.. zuul:autojob:: tox-docs +.. zuul:autojob:: tox-linters +.. zuul:autojob:: tox-pep8 +.. zuul:autojob:: tox-cover +.. zuul:autojob:: tox-bashate +.. zuul:autojob:: tox-nodejs-npm +.. zuul:autojob:: build-python-release +.. zuul:autojob:: python-upload-pypi +.. zuul:autojob:: build-sphinx-docs +.. zuul:autojob:: build-reno-releasenotes diff --git a/doc/source/python-roles.rst b/doc/source/python-roles.rst new file mode 100644 index 000000000..f7c8aacfc --- /dev/null +++ b/doc/source/python-roles.rst @@ -0,0 +1,21 @@ +Python Roles +============ + +.. zuul:autorole:: build-python-release +.. zuul:autorole:: build-releasenotes +.. zuul:autorole:: ensure-babel +.. zuul:autorole:: ensure-python +.. zuul:autorole:: ensure-sphinx +.. zuul:autorole:: ensure-tox +.. zuul:autorole:: ensure-twine +.. zuul:autorole:: fetch-coverage-output +.. zuul:autorole:: fetch-python-sdist-output +.. zuul:autorole:: fetch-sphinx-output +.. zuul:autorole:: fetch-sphinx-tarball +.. zuul:autorole:: fetch-subunit-output +.. zuul:autorole:: fetch-tox-output +.. zuul:autorole:: find-constraints +.. zuul:autorole:: install-if-python +.. zuul:autorole:: sphinx +.. zuul:autorole:: tox +.. zuul:autorole:: upload-pypi diff --git a/doc/source/roles.rst b/doc/source/roles.rst index 5baa1e4e8..debc6534d 100644 --- a/doc/source/roles.rst +++ b/doc/source/roles.rst @@ -1,4 +1,17 @@ Roles ===== -.. zuul:autoroles:: +.. toctree:: + :maxdepth: 1 + + general-roles + log-roles + afs-roles + container-roles + deprecated-roles + galaxy-roles + js-roles + launchpad-roles + puppet-roles + python-roles + translation-roles diff --git a/doc/source/translation-roles.rst b/doc/source/translation-roles.rst new file mode 100644 index 000000000..e81a3f439 --- /dev/null +++ b/doc/source/translation-roles.rst @@ -0,0 +1,4 @@ +Translation Roles +================= + +.. zuul:autorole:: fetch-translation-output diff --git a/tools/check_jobs_documented.py b/tools/check_jobs_documented.py new file mode 100755 index 000000000..d39c0c9e8 --- /dev/null +++ b/tools/check_jobs_documented.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +# +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Ensure that all jobs and roles appear in the documentation. + +import os +import re +import sys +import yaml + + +class ZuulSafeLoader(yaml.SafeLoader): + + def __init__(self, *args, **kwargs): + super(ZuulSafeLoader, self).__init__(*args, **kwargs) + self.add_multi_constructor('!encrypted/', self.construct_encrypted) + + @classmethod + def construct_encrypted(cls, loader, tag_suffix, node): + return loader.construct_sequence(node) + + +class Layout(object): + def __init__(self): + self.jobs = [] + + +class ZuulConfig(object): + def find_zuul_yaml(self): + root = os.getcwd() + while root: + for fn in ['zuul.yaml', '.zuul.yaml', 'zuul.d', '.zuul.d']: + path = os.path.join(root, fn) + if os.path.exists(path): + return path + root = os.path.split(root)[0] + raise Exception( + "Unable to find zuul config in zuul.yaml, .zuul.yaml," + " zuul.d or .zuul.d") + + def parse_zuul_yaml(self, path): + with open(path) as f: + data = yaml.load(f, Loader=ZuulSafeLoader) + layout = Layout() + for obj in data: + if 'job' in obj: + layout.jobs.append(obj['job']) + return layout + + def parse_zuul_d(self, path): + layout = Layout() + for conf in os.listdir(path): + with open(os.path.join(path, conf)) as f: + data = yaml.load(f, Loader=ZuulSafeLoader) + for obj in data: + if 'job' in obj: + layout.jobs.append(obj['job']) + return layout + + def parse_zuul_layout(self): + path = self.find_zuul_yaml() + if path.endswith('zuul.d'): + layout = self.parse_zuul_d(path) + else: + layout = self.parse_zuul_yaml(path) + return layout + + def __init__(self): + self.layout = self.parse_zuul_layout() + + +class Docs(object): + def __init__(self): + self.jobs = set() + self.roles = set() + self.autojobs = False + self.autoroles = False + self.walk(os.path.join(os.getcwd(), 'doc', 'source')) + + def walk(self, path): + for root, dirs, files in os.walk(path): + for fn in files: + if fn.endswith('.rst'): + with open(os.path.join(root, fn)) as f: + for line in f: + m = re.match(r'.*\.\. zuul:job:: (.*)$', line) + if m: + self.jobs.add(m.group(1)) + m = re.match(r'.*\.\. zuul:autojob:: (.*)$', line) + if m: + self.jobs.add(m.group(1)) + m = re.match(r'.*\.\. zuul:autojobs::.*$', line) + if m: + self.autojobs = True + m = re.match(r'.*\.\. zuul:role:: (.*)$', line) + if m: + self.roles.add(m.group(1)) + m = re.match(r'.*\.\. zuul:autorole:: (.*)$', line) + if m: + self.roles.add(m.group(1)) + m = re.match(r'.*\.\. zuul:autoroles::.*$', line) + if m: + self.autoroles = True + + +class Roles(object): + def __init__(self): + self.roles = set() + self.walk(os.path.join(os.getcwd(), 'roles')) + + def walk(self, path): + for role in os.listdir(path): + if os.path.isdir(os.path.join(path, role, 'tasks')): + self.roles.add(role) + + +z = ZuulConfig() +r = Roles() +d = Docs() + +ret = 0 +for role in r.roles: + if role not in d.roles: + print("Role %s not included in document tree" % (role,)) + ret = 1 +for job in [x['name'] for x in z.layout.jobs]: + if job not in d.jobs: + print("Job %s not included in document tree" % (job,)) + ret = 1 + +sys.exit(ret) diff --git a/tox.ini b/tox.ini index b53e98bad..6f30ba105 100644 --- a/tox.ini +++ b/tox.ini @@ -46,6 +46,7 @@ commands = # Ansible Syntax Check bash -c "find playbooks -type f -regex '.*.ya?ml' -exec \ ansible-playbook --syntax-check -i {toxinidir}/tests/inventory \{\} + > /dev/null" + {toxinidir}/tools/check_jobs_documented.py [testenv:venv] commands = {posargs}