Support multiple repo for document source

- A single primary repo must be specified which
  holds the site_definition.yaml file.
- Zero or more auxiliary repos can be specified which
  have additional documents used in the site definition.
- Collected documents are written to a file named after their
  source repo. Collection must always be given a output directory
  rather than a single file now.

Change-Id: Iceda4da18c4df45d917d88a49144e39e3f1743ed

.gitignore

@@ -0,0 +1,7 @@
+__pycache__
+.tox
+.eggs
+pegleg.egg-info
+/ChangeLog
+/AUTHORS
+*.swp

src/bin/pegleg/.gitignore

@@ -1,6 +0,0 @@
-__pycache__
-/.tox
-/.eggs
-/pegleg.egg-info
-/ChangeLog
-/AUTHORS

src/bin/pegleg/pegleg/cli.py

@@ -1,4 +1,6 @@
 from . import engine
+from pegleg import config
+
 import click
 import logging
 import sys
@@ -13,14 +15,13 @@ CONTEXT_SETTINGS = {
 
 
 @click.group(context_settings=CONTEXT_SETTINGS)
-@click.pass_context
 @click.option(
     '-v',
     '--verbose',
     is_flag=bool,
     default=False,
     help='Enable debug logging')
-def main(ctx, *, verbose):
+def main(*, verbose):
     if verbose:
         log_level = logging.DEBUG
     else:
@@ -29,21 +30,36 @@ def main(ctx, *, verbose):
 
 
 @main.group(help='Commands related to sites')
-def site():
-    pass
+@click.option(
+    '-p',
+    '--primary',
+    'primary_repo',
+    required=True,
+    help=
+    'Path to the root of the primary (containing site_definition.yaml) repo.')
+@click.option(
+    '-a',
+    '--auxiliary',
+    'aux_repo',
+    multiple=True,
+    help='Path to the root of an auxiliary repo.')
+def site(primary_repo, aux_repo):
+    config.set_primary_repo(primary_repo)
+    config.set_auxiliary_repo_list(aux_repo or [])
 
 
 @site.command(help='Output complete config for one site')
 @click.option(
-    '-o',
-    '--output',
-    'output_stream',
-    type=click.File(mode='w'),
+    '-s',
+    '--save-location',
+    'save_location',
+    type=click.Path(
+        file_okay=False, dir_okay=True, writable=True, resolve_path=True),
     default=sys.stdout,
     help='Where to output')
 @click.argument('site_name')
-def collect(*, output_stream, site_name):
-    engine.site.collect(site_name, output_stream)
+def collect(*, save_location, site_name):
+    engine.site.collect(site_name, save_location)
 
 
 @site.command(help='Find sites impacted by changed files')
@@ -161,5 +177,24 @@ def site_type(*, revision, site_type):
 
 @LINT_OPTION
 @main.command(help='Sanity checks for repository content')
-def lint(*, fail_on_missing_sub_src):
-    engine.lint.full(fail_on_missing_sub_src)
+@click.option(
+    '-p',
+    '--primary',
+    'primary_repo',
+    required=True,
+    help=
+    'Path to the root of the primary (containing site_definition.yaml) repo.')
+@click.option(
+    '-a',
+    '--auxiliary',
+    'aux_repo',
+    multiple=True,
+    help='Path to the root of a auxiliary repo.')
+def lint(*, fail_on_missing_sub_src, primary_repo, aux_repo):
+    config.set_primary_repo(primary_repo)
+    config.set_auxiliary_repo_list(aux_repo or [])
+    warns = engine.lint.full(fail_on_missing_sub_src)
+    if warns:
+        click.echo("Linting passed, but produced some warnings.")
+        for w in warns:
+            click.echo(w)

src/bin/pegleg/pegleg/config.py

@@ -0,0 +1,41 @@
+from copy import copy
+
+try:
+    if GLOBAL_CONTEXT:
+        pass
+except NameError:
+    GLOBAL_CONTEXT = {
+        'primary_repo': './',
+        'aux_repos': [],
+    }
+
+
+def get_primary_repo():
+    return GLOBAL_CONTEXT['primary_repo']
+
+
+def set_primary_repo(r):
+    GLOBAL_CONTEXT['primary_repo'] = r
+
+
+def set_auxiliary_repo_list(a):
+    GLOBAL_CONTEXT['aux_repos'] = copy(a)
+
+
+def add_auxiliary_repo(a):
+    GLOBAL_CONTEXT['aux_repos'].append(a)
+
+
+def get_auxiliary_repo_list():
+    return GLOBAL_CONTEXT['aux_repos']
+
+
+def each_auxiliary_repo():
+    for a in GLOBAL_CONTEXT['aux_repos']:
+        yield a
+
+
+def all_repos():
+    repos = [get_primary_repo()]
+    repos.extend(get_auxiliary_repo_list())
+    return repos

src/bin/pegleg/pegleg/engine/lint.py

@@ -1,11 +1,12 @@
-from pegleg.engine import util
 import click
-import jsonschema
 import logging
 import os
 import pkg_resources
 import yaml
 
+from pegleg.engine import util
+from pegleg import config
+
 __all__ = ['full']
 
 LOG = logging.getLogger(__name__)
@@ -19,11 +20,14 @@ DECKHAND_SCHEMAS = {
 
 def full(fail_on_missing_sub_src=False):
     errors = []
-    errors.extend(_verify_no_unexpected_files())
+    warns = []
+    warns.extend(_verify_no_unexpected_files())
     errors.extend(_verify_file_contents())
     errors.extend(_verify_deckhand_render(fail_on_missing_sub_src))
     if errors:
-        raise click.ClickException('\n'.join(['Linting failed:'] + errors))
+        raise click.ClickException('\n'.join(
+            ['Linting failed:'] + errors + ['Linting warnings:'] + warns))
+    return warns
 
 
 def _verify_no_unexpected_files():
@@ -36,16 +40,16 @@ def _verify_no_unexpected_files():
     found_directories = util.files.existing_directories()
     LOG.debug('found_directories: %s', found_directories)
 
-    errors = []
+    msgs = []
     for unused_dir in sorted(found_directories - expected_directories):
-        errors.append('%s exists, but is unused' % unused_dir)
+        msgs.append('%s exists, but is unused' % unused_dir)
 
     for missing_dir in sorted(expected_directories - found_directories):
         if not missing_dir.endswith('common'):
-            errors.append(
+            msgs.append(
                 '%s was not found, but expected by manifest' % missing_dir)
 
-    return errors
+    return msgs
 
 
 def _verify_file_contents():
@@ -89,18 +93,6 @@ def _verify_document(document, schemas, filename):
         document.get('metadata', {}).get('name', '')
     ])
     errors = []
-    try:
-        jsonschema.validate(document, schemas['root'])
-        try:
-            jsonschema.validate(document['metadata'],
-                                schemas[document['metadata']['schema']])
-        except Exception as e:
-            errors.append('%s (document %s) failed Deckhand metadata schema '
-                          'validation: %s' % (filename, name, e))
-    except Exception as e:
-        errors.append(
-            '%s (document %s) failed Deckhand root schema validation: %s' %
-            (filename, name, e))
 
     layer = _layer(document)
     if layer is not None and layer != _expected_layer(filename):
@@ -147,8 +139,11 @@ def _layer(data):
 
 
 def _expected_layer(filename):
-    parts = os.path.normpath(filename).split(os.sep)
-    return parts[0]
+    for r in config.all_repos():
+        if filename.startswith(r + "/"):
+            partial_name = filename[len(r) + 1:]
+            parts = os.path.normpath(partial_name).split(os.sep)
+            return parts[0]
 
 
 def _load_schemas():

src/bin/pegleg/pegleg/engine/site.py

@@ -1,16 +1,36 @@
-from pegleg.engine import util
+import os
+import click
 import collections
 import csv
 import json
 import yaml
+import logging
 
+from pegleg.engine import util
 __all__ = ['collect', 'impacted', 'list_', 'show', 'render']
 
+LOG = logging.getLogger(__name__)
 
-def collect(site_name, output_stream):
-    for filename in util.definition.site_files(site_name):
-        with open(filename) as f:
-            output_stream.writelines(f.readlines())
+
+def collect(site_name, save_location):
+    try:
+        save_files = dict()
+        for (repo_base,
+             filename) in util.definition.site_files_by_repo(site_name):
+            repo_name = os.path.normpath(repo_base).split(os.sep)[-1]
+            if repo_name not in save_files:
+                save_files[repo_name] = open(
+                    os.path.join(save_location, repo_name + ".yaml"), "w")
+            LOG.debug("Collecting file %s to file %s" %
+                      (filename,
+                       os.path.join(save_location, repo_name + '.yaml')))
+            with open(filename) as f:
+                save_files[repo_name].writelines(f.readlines())
+    except Exception as ex:
+        raise click.ClickException("Error saving output: %s" % str(ex))
+    finally:
+        for f in save_files.values():
+            f.close()
 
 
 def impacted(input_stream, output_stream):
@@ -35,8 +55,7 @@ def render(site_name, output_stream):
 
     rendered_documents, errors = util.deckhand.deckhand_render(
         documents=documents)
-    for d in documents:
-        output_stream.writelines(yaml.dump(d))
+    yaml.dump_all(rendered_documents, output_stream, default_flow_style=False)
 
 
 def list_(output_stream):

src/bin/pegleg/pegleg/engine/util/definition.py

@@ -1,6 +1,8 @@
-from . import files
 import click
 
+from pegleg import config
+from . import files
+
 __all__ = [
     'create',
     'load',
@@ -31,19 +33,21 @@ def create(*, site_name, site_type, revision):
     files.dump(path(site_name), definition)
 
 
-def load(site):
-    return files.slurp(path(site))
+def load(site, primary_repo_base=None):
+    return files.slurp(path(site, primary_repo_base))
 
 
-def load_as_params(site_name):
-    definition = load(site_name)
+def load_as_params(site_name, primary_repo_base=None):
+    definition = load(site_name, primary_repo_base)
     params = definition.get('data', {})
     params['site_name'] = site_name
     return params
 
 
-def path(site_name):
-    return 'site/%s/site-definition.yaml' % site_name
+def path(site_name, primary_repo_base=None):
+    if not primary_repo_base:
+        primary_repo_base = config.get_primary_repo()
+    return '%s/site/%s/site-definition.yaml' % (primary_repo_base, site_name)
 
 
 def pluck(site_definition, key):
@@ -61,3 +65,14 @@ def site_files(site_name):
     for filename in files.search(files.directories_for(**params)):
         yield filename
     yield path(site_name)
+
+
+def site_files_by_repo(site_name):
+    """Yield tuples of repo_base, file_name."""
+    params = load_as_params(site_name)
+    dir_map = files.directories_for_each_repo(**params)
+    for repo, dl in dir_map.items():
+        for filename in files.search(dl):
+            yield (repo, filename)
+        if repo == config.get_primary_repo():
+            yield (repo, path(site_name))

src/bin/pegleg/pegleg/engine/util/files.py

@@ -1,6 +1,11 @@
 import click
 import os
 import yaml
+import logging
+
+from pegleg import config
+
+LOG = logging.getLogger(__name__)
 
 __all__ = [
     'all',
@@ -23,7 +28,10 @@ DIR_DEPTHS = {
 
 
 def all():
-    return search(DIR_DEPTHS.keys())
+    return search([
+        os.path.join(r, k) for r in config.all_repos()
+        for k in DIR_DEPTHS.keys()
+    ])
 
 
 def create_global_directories(revision):
@@ -83,7 +91,7 @@ def _create_tree(root_path, *, tree=FULL_STRUCTURE):
 
 
 def directories_for(*, site_name, revision, site_type):
-    return [
+    library_list = [
         _global_common_path(),
         _global_revision_path(revision),
         _site_type_common_path(site_type),
@@ -91,6 +99,32 @@ def directories_for(*, site_name, revision, site_type):
         _site_path(site_name),
     ]
 
+    return [
+        os.path.join(b, l) for b in config.all_repos() for l in library_list
+    ]
+
+
+def directories_for_each_repo(*, site_name, revision, site_type):
+    """Provide directories for each repo.
+
+    When producing bucketized output files, the documents collected
+    must be collated by repo. Provide the list of source directories
+    by repo.
+    """
+    library_list = [
+        _global_common_path(),
+        _global_revision_path(revision),
+        _site_type_common_path(site_type),
+        _site_type_revision_path(site_type, revision),
+        _site_path(site_name),
+    ]
+
+    dir_map = dict()
+    for r in config.all_repos():
+        dir_map[r] = [os.path.join(r, l) for l in library_list]
+
+    return dir_map
+
 
 def _global_common_path():
     return 'global/common'
@@ -112,24 +146,32 @@ def _site_path(site_name):
     return 'site/%s' % site_name
 
 
-def list_sites():
-    for path in os.listdir('site'):
-        joined_path = os.path.join('site', path)
+def list_sites(primary_repo_base=None):
+    """Get a list of site defintion directories in the primary repo."""
+    if not primary_repo_base:
+        primary_repo_base = config.get_primary_repo()
+    for path in os.listdir(os.path.join(primary_repo_base, 'site')):
+        joined_path = os.path.join(primary_repo_base, 'site', path)
         if os.path.isdir(joined_path):
             yield path
 
 
 def directory_for(*, path):
-    parts = os.path.normpath(path).split(os.sep)
-    depth = DIR_DEPTHS.get(parts[0])
-    if depth is not None:
-        return os.path.join(*parts[:depth + 1])
+    for r in config.all_repos():
+        if path.startswith(r + "/"):
+            partial_path = path[len(r) + 1:]
+            parts = os.path.normpath(partial_path).split(os.sep)
+            depth = DIR_DEPTHS.get(parts[0])
+            if depth is not None:
+                return os.path.join(r, *parts[:depth + 1])
 
 
 def existing_directories():
     directories = set()
-    for search_path, depth in DIR_DEPTHS.items():
-        directories.update(_recurse_subdirs(search_path, depth))
+    for r in config.all_repos():
+        for search_path, depth in DIR_DEPTHS.items():
+            directories.update(
+                _recurse_subdirs(os.path.join(r, search_path), depth))
     return directories
 
 
@@ -141,7 +183,7 @@ def slurp(path):
 
     with open(path) as f:
         try:
-            return yaml.load(f)
+            return yaml.safe_load(f)
         except Exception as e:
             raise click.ClickException('Failed to parse %s:\n%s' % (path, e))
 
@@ -158,18 +200,24 @@ def dump(path, data):
 
 def _recurse_subdirs(search_path, depth):
     directories = set()
-    for path in os.listdir(search_path):
-        joined_path = os.path.join(search_path, path)
-        if os.path.isdir(joined_path):
-            if depth == 1:
-                directories.add(joined_path)
-            else:
-                directories.update(_recurse_subdirs(joined_path, depth - 1))
+    try:
+        for path in os.listdir(search_path):
+            joined_path = os.path.join(search_path, path)
+            if os.path.isdir(joined_path):
+                if depth == 1:
+                    directories.add(joined_path)
+                else:
+                    directories.update(
+                        _recurse_subdirs(joined_path, depth - 1))
+    except FileNotFoundError:
+        pass
     return directories
 
 
 def search(search_paths):
     for search_path in search_paths:
+        LOG.debug("Recursively collecting YAMLs from %s" % search_path)
         for root, _dirs, filenames in os.walk(search_path):
             for filename in filenames:
-                yield os.path.join(root, filename)
+                if filename.endswith(".yaml"):
+                    yield os.path.join(root, filename)

tools/pegleg.sh

@@ -8,14 +8,16 @@ realpath() {
 
 SCRIPT_DIR=$(realpath "$(dirname "${0}")")
 SOURCE_DIR=${SCRIPT_DIR}/pegleg
-if [ -d "$PWD/global" ]; then
-  WORKSPACE="$PWD"
-else
-  WORKSPACE=$(realpath "${SCRIPT_DIR}/..")
+if [ -z "${WORKSPACE}" ]; then
+  WORKSPACE="/"
 fi
 
 IMAGE=${IMAGE:-quay.io/attcomdev/pegleg:latest}
 
+echo
+echo "== NOTE: Workspace $WORKSPACE  is available as /workspace in container context =="
+echo
+
 docker run --rm -t \
     --net=none \
     -v "${WORKSPACE}:/workspace" \