airship/pegleg
Code Issues Proposed changes

CLI: Add command to generate genesis bundle

Browse Source 
Added a pegleg cli command to build genesis.sh bundle for
a site deployment.
Pegleg imports promenade engine, and uses promenade to build
and encrypt the genesis.sh deployment bundle.

Change-Id: I1a489459b2c56b7b53018c32aab5e6550c69e1d2
This commit is contained in:
Ahmad Mahmoudi 2018-10-25 12:55:34 -05:00 committed by Lev Morgan
parent 50ce7a02e0
commit c4f25b4d4f
9 changed files with 392 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
doc/source/cli/cli.rst
View File

@ -612,6 +612,42 @@ Example:
secrets decrypt site1 -f \
/opt/security-manifests/site/site1/passwords/password1.yaml
genesis_bundle
--------------
Constructs genesis bundle based on a site configuration.
.. note::
This command requires the environment variable PEGLEG_PASSPHRASE
to be set and at least 24 characters long, to be used for encrypting
genesis bundle data. PEGLEG_SALT must be set as well. There are no
constraints on its length, but at least 24 characters is recommended.
**-b / --build-dir** (Required).
Destination directory for the genesis bundle.
**--include-validators** (Optional). False by default.
A flag to request build genesis validation scripts as well.
Usage:
::
./pegleg.sh site <options> genesis_bundle <site_name> \
-b <build_locaton> -k <encryption_passphrase/key> --validators
Examples
^^^^^^^^
::
./pegleg.sh site -r ./site-manifests \
genesis_bundle site1 \
-b ../../site1_build \
-k yourEncryptionPassphrase \
--validators
generate
^^^^^^^^
@ -803,8 +839,9 @@ Where mandatory encrypted schema type is one of:
P002 - Deckhand rendering is expected to complete without errors.
P003 - All repos contain expected directories.
.. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/rendering.html
.. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/validation.html
.. _Deckhand: https://airship-deckhand.readthedocs.io/en/latest/users/rendering.html
.. _Deckhand Validations: https://airship-deckhand.readthedocs.io/en/latest/overview.html#validation
.. _Pegleg Managed Documents: https://airship-specs.readthedocs.io/en/latest/specs/approved/pegleg-secrets.html#peglegmanageddocument
.. _Shipyard: https://github.com/openstack/airship-shipyard
.. _CLI documentation: https://airship-shipyard.readthedocs.io/en/latest/CLI.html#openstack-keystone-authorization-environment-variables
@ -878,4 +915,4 @@ Example with length specified:
::
./pegleg.sh generate salt -l <length>
./pegleg.sh generate salt -l <length>

9
doc/source/exceptions.rst
View File

@ -68,10 +68,19 @@ PKI Exceptions
--------------
.. autoexception:: pegleg.engine.exceptions.IncompletePKIPairError
Genesis Bundle Exceptions
-------------------------
.. autoexception:: pegleg.engine.exceptions.GenesisBundleEncryptionException
:members:
:show-inheritance:
:undoc-members:
.. autoexception:: pegleg.engine.exceptions.GenesisBundleGenerateException
:members:
:show-inheritance:
Passphrase Exceptions
---------------------

51
pegleg/cli.py
View File

@ -14,13 +14,16 @@
import functools
import logging
import os
import sys
import click
from pegleg import config
from pegleg import engine
from pegleg.engine import bundle
from pegleg.engine import catalog
from pegleg.engine.util.pegleg_secret_management import PeglegSecretManagement
from pegleg.engine.util.shipyard_helper import ShipyardHelper
LOG = logging.getLogger(__name__)
@ -412,6 +415,54 @@ def generate_pki(site_name, author):
click.echo("Generated PKI files written to:\n%s" % '\n'.join(output_paths))
@site.command(
'genesis_bundle',
help='Construct the genesis deployment bundle.')
@click.option(
'-b',
'--build-dir',
'build_dir',
type=click.Path(file_okay=False, dir_okay=True, resolve_path=True),
required=True,
help='Destination directory to store the genesis bundle.')
@click.option(
'--include-validators',
'validators',
is_flag=True,
default=False,
help='A flag to request generate genesis validation scripts in addition '
'to genesis.sh script.')
@SITE_REPOSITORY_ARGUMENT
def genesis_bundle(*, build_dir, validators, site_name):
prom_encryption_key = os.environ.get("PROMENADE_ENCRYPTION_KEY")
peg_encryption_key = os.environ.get("PEGLEG_PASSPHRASE")
encryption_key = None
if (prom_encryption_key and len(prom_encryption_key) > 24 and
peg_encryption_key and len(peg_encryption_key) > 24):
click.echo("WARNING: PROMENADE_ENCRYPTION_KEY is deprecated, "
"using PEGLEG_PASSPHRASE instead", err=True)
config.set_passphrase(peg_encryption_key)
encryption_key = peg_encryption_key
elif prom_encryption_key and len(prom_encryption_key) > 24:
click.echo("ERROR: PROMENADE_ENCRYPTION_KEY is deprecated, "
"use PEGLEG_PASSPHRASE instead", err=True)
raise click.ClickException("ERROR: PEGLEG_PASSPHRASE must be set "
"and at least 24 characters long.")
elif peg_encryption_key and len(peg_encryption_key) > 24:
config.set_passphrase(peg_encryption_key)
encryption_key = peg_encryption_key
else:
raise click.ClickException("ERROR: PEGLEG_PASSPHRASE must be set "
"and at least 24 characters long.")
PeglegSecretManagement.check_environment()
bundle.build_genesis(build_dir,
encryption_key,
validators,
logging.DEBUG == LOG.getEffectiveLevel(),
site_name)
@main.group(help='Commands related to types')
@MAIN_REPOSITORY_OPTION
@REPOSITORY_CLONE_PATH_OPTION

24
pegleg/config.py
View File

@ -26,7 +26,9 @@ except NameError:
'clone_path': None,
'site_path': 'site',
'site_rev': None,
'type_path': 'type'
'type_path': 'type',
'passphrase': None,
'salt': None
}
@ -147,3 +149,23 @@ def set_rel_type_path(p):
"""Set the relative type path name."""
p = p or 'type'
GLOBAL_CONTEXT['type_path'] = p
def set_passphrase(p):
"""Set the passphrase for encryption and decryption."""
GLOBAL_CONTEXT['passphrase'] = p
def get_passphrase():
"""Get the passphrase for encryption and decryption."""
return GLOBAL_CONTEXT['passphrase']
def set_salt(p):
"""Set the salt for encryption and decryption."""
GLOBAL_CONTEXT['salt'] = p
def get_salt():
"""Get the salt for encryption and decryption."""
return GLOBAL_CONTEXT['salt']

92
pegleg/engine/bundle.py Normal file
View File

@ -0,0 +1,92 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
import stat
import click
from pegleg.engine.exceptions import GenesisBundleEncryptionException
from pegleg.engine.exceptions import GenesisBundleGenerateException
from pegleg.engine import util
from pegleg.engine.util.pegleg_secret_management import PeglegSecretManagement
from promenade.builder import Builder
from promenade.config import Configuration
from promenade import exceptions
LOG = logging.getLogger(__name__)
__all__ = [
'build_genesis',
]
def build_genesis(build_path, encryption_key, validators, debug, site_name):
"""
Build the genesis deployment bundle, and store it in ``build_path``.
Build the genesis.sh script, base65-encode, encrypt and embed the
site configuration source documents in genesis.sh script.
If ``validators`` flag should be True, build the bundle validator
scripts as well.
Store the built deployment bundle in `build_path`.
:param str build_path: Directory path of the built genesis deployment
bundle
:param str encryption_key: Key to use to encrypt the bundled site
configuration in genesis.sh script.
:param bool validators: Whether to generate validator scripts
:param int debug: pegleg debug level to pass to promenade engine
for logging.
:return: None
"""
# Raise an error if the build path exists. We don't want to overwrite it.
if os.path.isdir(build_path):
raise click.ClickException(
"{} already exists, remove it or specify a new "
"directory.".format(build_path))
# Get the list of config files
LOG.info('=== Building bootstrap scripts ===')
# Copy the site config, and site secrets to build directory
os.mkdir(build_path)
os.chmod(build_path, os.stat(build_path).st_mode | stat.S_IRWXU |
stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH)
documents = util.definition.documents_for_site(site_name)
secret_manager = PeglegSecretManagement(docs=documents)
documents = secret_manager.get_decrypted_secrets()
try:
# Use the promenade engine to build and encrypt the genesis bundle
c = Configuration(
documents=documents,
debug=debug,
substitute=True,
allow_missing_substitutions=False,
leave_kubectl=False)
if c.get_path('EncryptionPolicy:scripts.genesis') and encryption_key:
os.environ['PROMENADE_ENCRYPTION_KEY'] = encryption_key
os.environ['PEGLEG_PASSPHRASE'] = encryption_key
Builder(c, validators=validators).build_all(output_dir=build_path)
else:
raise GenesisBundleEncryptionException()
except exceptions.PromenadeException as e:
LOG.error('Build genesis bundle failed! {}.'.format(
e.display(debug=debug)))
raise GenesisBundleGenerateException()
LOG.info('=== Done! ===')

16
pegleg/engine/exceptions.py
View File

@ -86,3 +86,19 @@ class PassphraseCatalogNotFoundException(PeglegBaseException):
"""Failed to find Catalog for Passphrases generation."""
message = ('Could not find the Passphrase Catalog to generate '
'the site Passphrases!')
class GenesisBundleEncryptionException(PeglegBaseException):
"""Exception raised when encryption of the genesis bundle fails."""
message = 'Encryption is required for genesis bundle, but no encryption ' \
'policy or key is specified.'
class GenesisBundleGenerateException(PeglegBaseException):
"""
Exception raised when pormenade engine fails to build the genesis
bundle.
"""
message = 'Bundle generation failed on deckhand validation.'

21
pegleg/engine/util/pegleg_secret_management.py
View File

@ -19,6 +19,7 @@ import re
import click
import yaml
from pegleg import config
from pegleg.engine.util.encryption import decrypt
from pegleg.engine.util.encryption import encrypt
from pegleg.engine.util import files
@ -47,10 +48,11 @@ class PeglegSecretManagement(object):
raise ValueError('Either `file_path` or `docs` must be '
'specified.')
if generated and not (author and catalog):
if generated and not (catalog and author):
raise ValueError("If the document is generated, author and "
"catalog must be specified.")
self.__check_environment()
self.check_environment()
self.file_path = file_path
self.documents = list()
self._generated = generated
@ -68,8 +70,17 @@ class PeglegSecretManagement(object):
self._author = author
self.passphrase = os.environ.get(ENV_PASSPHRASE).encode()
self.salt = os.environ.get(ENV_SALT).encode()
if config.get_passphrase() and config.get_salt():
self.passphrase = config.get_passphrase()
self.salt = config.get_salt()
elif config.get_passphrase() or config.get_salt():
raise ValueError("ERROR: Pegleg configuration must either have "
"both a passphrase and a salt or neither.")
else:
self.passphrase = os.environ.get(ENV_PASSPHRASE).encode()
self.salt = os.environ.get(ENV_SALT).encode()
config.set_passphrase(self.passphrase)
config.set_salt(self.salt)
def __iter__(self):
"""
@ -79,7 +90,7 @@ class PeglegSecretManagement(object):
return (doc.pegleg_document for doc in self.documents)
@staticmethod
def __check_environment():
def check_environment():
"""
Validate required environment variables for encryption or decryption.

1
requirements.txt
View File

@ -9,3 +9,4 @@ python-dateutil==2.7.3
rstr==2.2.6
git+https://github.com/openstack/airship-deckhand.git@49ad9f38842f7f1ecb86d907d86d332f8186eb8c
git+https://github.com/openstack/airship-shipyard.git@44f7022df6438de541501c2fdd5c46df198b82bf#egg=shipyard_client&subdirectory=src/bin/shipyard_client
git+https://github.com/openstack/airship-promenade.git@a6e8fdbe22bd153c78a008b92cd5d1c245bc63e3

144
tests/unit/engine/test_build_genesis_bundle.py Normal file
View File

@ -0,0 +1,144 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
import mock
import pytest
import yaml
from pegleg import config
from pegleg.engine import bundle
from pegleg.engine.exceptions import GenesisBundleEncryptionException
from pegleg.engine.exceptions import GenesisBundleGenerateException
from pegleg.engine.util import files
from pegleg.engine.util.pegleg_secret_management import ENV_PASSPHRASE
from pegleg.engine.util.pegleg_secret_management import ENV_SALT
from tests.unit.fixtures import temp_path
SITE_DEFINITION = """
---
# High-level pegleg site definition file
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
# NEWSITE-CHANGEME: Replace with the site name
name: test_site
storagePolicy: cleartext
data:
# The type layer this site will delpoy with. Type layer is found in the
# type folder.
site_type: foundry
...
"""
SITE_CONFIG_DATA = """
---
schema: promenade/EncryptionPolicy/v1
metadata:
schema: metadata/Document/v1
name: encryption-policy
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
scripts:
genesis:
gpg: {}
join:
gpg: {}
---
schema: deckhand/LayeringPolicy/v1
metadata:
schema: metadata/Control/v1
name: layering-policy
data:
layerOrder:
- global
- type
- site
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: ceph_swift_keystone_password
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: ABAgagajajkb839215387
...
"""
@mock.patch.dict(os.environ, {
ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
ENV_SALT: 'MySecretSalt'
})
def test_no_encryption_key(temp_path):
# Write the test data to temp file
config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA))
base_config_dir = os.path.join(temp_path, 'config_dir')
config.set_site_repo(base_config_dir)
config_dir = os.path.join(base_config_dir, 'site', 'test_site')
config_path = os.path.join(config_dir, 'config_file.yaml')
build_dir = os.path.join(temp_path, 'build_dir')
os.makedirs(config_dir)
files.write(config_path, config_data)
files.write(os.path.join(config_dir, "site-definition.yaml"),
yaml.safe_load_all(SITE_DEFINITION))
with pytest.raises(GenesisBundleEncryptionException,
match=r'.*no encryption policy or key is specified.*'):
bundle.build_genesis(build_path=build_dir,
encryption_key=None,
validators=False,
debug=logging.ERROR,
site_name="test_site")
@mock.patch.dict(os.environ, {
ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
ENV_SALT: 'MySecretSalt'
})
def test_failed_deckhand_validation(temp_path):
# Write the test data to temp file
config_data = list(yaml.safe_load_all(SITE_CONFIG_DATA))
base_config_dir = os.path.join(temp_path, 'config_dir')
config.set_site_repo(base_config_dir)
config_dir = os.path.join(base_config_dir, 'site', 'test_site')
config_path = os.path.join(config_dir, 'config_file.yaml')
build_dir = os.path.join(temp_path, 'build_dir')
os.makedirs(config_dir)
files.write(config_path, config_data)
files.write(os.path.join(config_dir, "site-definition.yaml"),
yaml.safe_load_all(SITE_DEFINITION))
key = 'MyverYSecretEncryptionKey382803'
with pytest.raises(GenesisBundleGenerateException,
match=r'.*failed on deckhand validation.*'):
bundle.build_genesis(build_path=build_dir,
encryption_key=key,
validators=False,
debug=logging.ERROR,
site_name="test_site")