Only collect/parse Deckhand-formatted documents for processing

This patch set changes Pegleg in two similar ways:

1) Ignore certain types of files altogether:
   - those located in hidden folders
   - those prefixed with "." (files like .zuul.yaml)
2) Only read Deckhand-formatted documents for lint/collect/etc.
   commands as Pegleg need not consider other types of documents
   (it separately reads the site-definition.yaml for internal
    processing still).

The tools/ subfolder is also ignored as it can contain
.yaml files which are not Deckhand-formatted documents,
so need not be processed by pegleg.engine.

Change-Id: I8996b5d430cf893122af648ef8e5805b36c1bfd9

pegleg/cli.py

@@ -385,7 +385,7 @@ def secrets():
     'author',
     required=True,
     help='Identifier for the program or person who is encrypting the secrets '
-         'documents')
+    'documents')
 @click.argument('site_name')
 def encrypt(*, save_location, author, site_name):
     engine.repository.process_repositories(site_name)

pegleg/engine/lint.py

@@ -18,7 +18,6 @@ import os
 import pkg_resources
 import shutil
 import textwrap
-import yaml
 
 from prettytable import PrettyTable
 
@@ -223,16 +222,16 @@ def _verify_single_file(filename, schemas):
             errors.append((FILE_MISSING_YAML_DOCUMENT_HEADER,
                            '%s does not begin with YAML beginning of document '
                            'marker "---".' % filename))
-        f.seek(0)
-        documents = []
-        try:
-            documents = list(yaml.safe_load_all(f))
-        except Exception as e:
-            errors.append((FILE_CONTAINS_INVALID_YAML,
-                           '%s is not valid yaml: %s' % (filename, e)))
 
-        for document in documents:
+    documents = []
-            errors.extend(_verify_document(document, schemas, filename))
+    try:
+        documents = util.files.read(filename)
+    except Exception as e:
+        errors.append((FILE_CONTAINS_INVALID_YAML,
+                       '%s is not valid yaml: %s' % (filename, e)))
+
+    for document in documents:
+        errors.extend(_verify_document(document, schemas, filename))
 
     return errors
 

pegleg/engine/util/definition.py

@@ -15,7 +15,6 @@
 import os
 
 import click
-import yaml
 
 from pegleg import config
 from pegleg.engine.util import files
@@ -52,6 +51,7 @@ def load_as_params(site_name, *fields, primary_repo_base=None):
 
 
 def path(site_name, primary_repo_base=None):
+    """Retrieve path to the site-definition.yaml file for ``site_name``."""
     if not primary_repo_base:
         primary_repo_base = config.get_site_repo()
     return os.path.join(primary_repo_base, 'site', site_name,
@@ -100,8 +100,7 @@ def documents_for_each_site():
         paths = files.directories_for(**params)
         filenames = set(files.search(paths))
         for filename in filenames:
-            with open(filename) as f:
+            documents[sitename].extend(files.read(filename))
-                documents[sitename].extend(list(yaml.safe_load_all(f)))
 
     return documents
 
@@ -122,7 +121,6 @@ def documents_for_site(sitename):
     paths = files.directories_for(**params)
     filenames = set(files.search(paths))
     for filename in filenames:
-        with open(filename) as f:
+        documents.extend(files.read(filename))
-            documents.extend(list(yaml.safe_load_all(f)))
 
     return documents

pegleg/engine/util/files.py

@@ -18,6 +18,7 @@ import yaml
 import logging
 
 from pegleg import config
+from pegleg.engine.util import pegleg_managed_document as md
 
 LOG = logging.getLogger(__name__)
 
@@ -248,9 +249,35 @@ def read(path):
             '{} not found. Pegleg must be run from the root of a '
             'configuration repository.'.format(path))
 
+    def is_deckhand_document(document):
+        # Deckhand documents only consist of control and application
+        # documents.
+        valid_schemas = ('metadata/Control', 'metadata/Document')
+        if isinstance(document, dict):
+            schema = document.get('metadata', {}).get('schema', '')
+            # NOTE(felipemonteiro): The Pegleg site-definition.yaml is a
+            # Deckhand-formatted document currently but probably shouldn't
+            # be, because it has no business being in Deckhand. As such,
+            # treat it as a special case.
+            if "SiteDefinition" in document.get('schema', ''):
+                return False
+            if any(schema.startswith(x) for x in valid_schemas):
+                return True
+            else:
+                LOG.debug('Document with schema=%s is not a valid Deckhand '
+                          'schema. Ignoring it.', schema)
+        return False
+
+    def is_pegleg_managed_document(document):
+        return md.PeglegManagedSecretsDocument.is_pegleg_managed_secret(
+            document)
+
     with open(path) as stream:
         try:
-            return list(yaml.safe_load_all(stream))
+            return [
+                d for d in yaml.safe_load_all(stream)
+                if is_deckhand_document(d) or is_pegleg_managed_document(d)
+            ]
         except yaml.YAMLError as e:
             raise click.ClickException('Failed to parse %s:\n%s' % (path, e))
 
@@ -296,10 +323,25 @@ def _recurse_subdirs(search_path, depth):
 
 
 def search(search_paths):
+    if not isinstance(search_paths, (list, tuple)):
+        search_paths = [search_paths]
+
     for search_path in search_paths:
         LOG.debug("Recursively collecting YAMLs from %s" % search_path)
-        for root, _dirs, filenames in os.walk(search_path):
+        for root, _, filenames in os.walk(search_path):
+
+            # Ignore hidden folders like .tox or .git for faster processing.
+            if os.path.basename(root).startswith("."):
+                continue
+            # Skip over anything in tools/ because it will never contain valid
+            # Pegleg-owned manifest documents.
+            if "tools" in root.split("/"):
+                continue
+
             for filename in filenames:
+                # Ignore files like .zuul.yaml.
+                if filename.startswith("."):
+                    continue
                 if filename.endswith(".yaml"):
                     yield os.path.join(root, filename)
 

pegleg/engine/util/pegleg_secret_management.py

@@ -44,8 +44,7 @@ class PeglegSecretManagement():
 
         if all([file_path, docs]) or \
             not any([file_path, docs]):
-            raise ValueError(
+            raise ValueError('Either `file_path` or `docs` must be specified.')
-                'Either `file_path` or `docs` must be specified.')
 
         self.__check_environment()
         self.file_path = file_path
@@ -73,7 +72,7 @@ class PeglegSecretManagement():
         # Verify that passphrase environment variable is defined and is longer
         # than 24 characters.
         if not os.environ.get(ENV_PASSPHRASE) or not re.match(
-            PASSPHRASE_PATTERN, os.environ.get(ENV_PASSPHRASE)):
+                PASSPHRASE_PATTERN, os.environ.get(ENV_PASSPHRASE)):
             raise click.ClickException(
                 'Environment variable {} is not defined or '
                 'is not at least 24-character long.'.format(ENV_PASSPHRASE))
@@ -154,8 +153,7 @@ class PeglegSecretManagement():
             # do not decrypt already decrypted data
             if doc.is_encrypted():
                 doc.set_secret(
-                    decrypt(doc.get_secret(),
+                    decrypt(doc.get_secret(), self.passphrase,
-                            self.passphrase,
                             self.salt).decode())
                 doc.set_decrypted()
             doc_list.append(doc.embedded_document)

tests/unit/engine/test_encryption.py

@@ -30,7 +30,6 @@ from pegleg.engine.util.pegleg_secret_management import ENV_SALT
 from tests.unit.fixtures import temp_path
 from pegleg.engine.util import files
 
-
 TEST_DATA = """
 ---
 schema: deckhand/Passphrase/v1
@@ -60,22 +59,24 @@ def test_encrypt_and_decrypt():
 
 
 @mock.patch.dict(os.environ, {
-    ENV_PASSPHRASE:'aShortPassphrase',
+    ENV_PASSPHRASE: 'aShortPassphrase',
-    ENV_SALT: 'MySecretSalt'})
+    ENV_SALT: 'MySecretSalt'
+})
 def test_short_passphrase():
-    with pytest.raises(click.ClickException,
+    with pytest.raises(
-                       match=r'.*is not at least 24-character long.*'):
+            click.ClickException,
+            match=r'.*is not at least 24-character long.*'):
         PeglegSecretManagement('file_path')
 
 
-def test_PeglegManagedDocument():
+def test_pegleg_secret_management_constructor():
     test_data = yaml.load(TEST_DATA)
     doc = PeglegManagedSecretsDocument(test_data)
-    assert doc.is_storage_policy_encrypted() is True
+    assert doc.is_storage_policy_encrypted()
-    assert doc.is_encrypted() is False
+    assert not doc.is_encrypted()
 
 
-def test_PeglegSecretManagement():
+def test_pegleg_secret_management_constructor_with_invalid_arguments():
     with pytest.raises(ValueError) as err_info:
         PeglegSecretManagement(file_path=None, docs=None)
     assert 'Either `file_path` or `docs` must be specified.' in str(
@@ -87,40 +88,24 @@ def test_PeglegSecretManagement():
 
 
 @mock.patch.dict(os.environ, {
-    ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC',
+    ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
-    ENV_SALT: 'MySecretSalt'})
+    ENV_SALT: 'MySecretSalt'
-def test_encrypt_file():
+})
-    # write the test data to temp file
+def test_encrypt_decrypt_using_file_path(temp_path):
-    test_data = yaml.load(TEST_DATA)
-    dir = tempfile.mkdtemp()
-    file_path = os.path.join(dir, 'secrets_file.yaml')
-    save_path = os.path.join(dir, 'encrypted_secrets_file.yaml')
-    with open(file_path, 'w') as stream:
-        yaml.dump(test_data,
-                  stream,
-                  explicit_start=True,
-                  explicit_end=True,
-                  default_flow_style=False)
-    # read back the secrets data file and encrypt it
-    doc_mgr = PeglegSecretManagement(file_path)
-    doc_mgr.encrypt_secrets(save_path, 'test_author')
-    doc = doc_mgr.documents[0]
-    assert doc.is_encrypted()
-    assert doc.data['encrypted']['by'] == 'test_author'
-
-
-@mock.patch.dict(os.environ, {
-    ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC',
-    ENV_SALT: 'MySecretSalt'})
-def test_encrypt_decrypt_file(temp_path):
     # write the test data to temp file
     test_data = list(yaml.safe_load_all(TEST_DATA))
     file_path = os.path.join(temp_path, 'secrets_file.yaml')
     files.write(file_path, test_data)
     save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml')
+
+    # encrypt documents and validate that they were encrypted
     doc_mgr = PeglegSecretManagement(file_path=file_path)
     doc_mgr.encrypt_secrets(save_path, 'test_author')
-    # read back the encrypted file
+    doc = doc_mgr.documents[0]
+    assert doc.is_encrypted()
+    assert doc.data['encrypted']['by'] == 'test_author'
+
+    # decrypt documents and validate that they were decrypted
     doc_mgr = PeglegSecretManagement(save_path)
     decrypted_data = doc_mgr.get_decrypted_secrets()
     assert test_data[0]['data'] == decrypted_data[0]['data']
@@ -128,23 +113,31 @@ def test_encrypt_decrypt_file(temp_path):
 
 
 @mock.patch.dict(os.environ, {
-    ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC',
+    ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
-    ENV_SALT: 'MySecretSalt'})
+    ENV_SALT: 'MySecretSalt'
-def test_decrypt_document(temp_path):
+})
+def test_encrypt_decrypt_using_docs(temp_path):
     # write the test data to temp file
     test_data = list(yaml.safe_load_all(TEST_DATA))
     save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml')
+
+    # encrypt documents and validate that they were encrypted
     doc_mgr = PeglegSecretManagement(docs=test_data)
     doc_mgr.encrypt_secrets(save_path, 'test_author')
+    doc = doc_mgr.documents[0]
+    assert doc.is_encrypted()
+    assert doc.data['encrypted']['by'] == 'test_author'
+
     # read back the encrypted file
     with open(save_path) as stream:
         encrypted_data = list(yaml.safe_load_all(stream))
-    # this time pass a list of dicts to peglegSecretManager
+
+    # decrypt documents and validate that they were decrypted
     doc_mgr = PeglegSecretManagement(docs=encrypted_data)
     decrypted_data = doc_mgr.get_decrypted_secrets()
     assert test_data[0]['data'] == decrypted_data[0]['data']
     assert test_data[0]['schema'] == decrypted_data[0]['schema']
-    assert test_data[0]['metadata']['name'] == decrypted_data[0][
+    assert test_data[0]['metadata']['name'] == decrypted_data[0]['metadata'][
-        'metadata']['name']
+        'name']
     assert test_data[0]['metadata']['storagePolicy'] == decrypted_data[0][
         'metadata']['storagePolicy']

tests/unit/engine/test_lint.py

@@ -125,23 +125,6 @@ def test_verify_deckhand_render_site_documents_separately(
                     'storagePolicy': 'cleartext'
                 },
                 'schema': 'deckhand/Passphrase/v1'
-            }, {
-                'data': {
-                    'site_type': sitename,
-                    'repositories': {
-                        'global': mock.ANY
-                    }
-                },
-                'metadata': {
-                    'layeringDefinition': {
-                        'abstract': False,
-                        'layer': 'site'
-                    },
-                    'name': sitename,
-                    'schema': 'metadata/Document/v1',
-                    'storagePolicy': 'cleartext'
-                },
-                'schema': 'pegleg/SiteDefinition/v1'
             }]
             expected_documents.extend(documents)
 

tests/unit/engine/util/test_files.py

@@ -0,0 +1,38 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from pegleg import config
+from pegleg.engine.util import files
+from tests.unit.fixtures import create_tmp_deployment_files
+
+
+class TestFileHelpers(object):
+    def test_read_compatible_file(self, create_tmp_deployment_files):
+        path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'secrets',
+                            'passphrases', 'cicd-passphrase.yaml')
+        documents = files.read(path)
+        assert 1 == len(documents)
+
+    def test_read_incompatible_file(self, create_tmp_deployment_files):
+        # NOTE(felipemonteiro): The Pegleg site-definition.yaml is a
+        # Deckhand-formatted document currently but probably shouldn't be,
+        # because it has no business being in Deckhand. As such, validate that
+        # it is ignored.
+        path = os.path.join(config.get_site_repo(), 'site', 'cicd',
+                            'site-definition.yaml')
+        documents = files.read(path)
+        assert not documents, ("Documents returned should be empty for "
+                               "site-definition.yaml")