airship/deckhand
Code Issues Proposed changes

Sorting/filtering for rendered-documents.

Browse Source 
This PS implements sorting and filtering for rendered-documents
endpoint, adds additional validations for sorting, filtering
and other layering scenarios, and updates rendered-documents
and buckets documentation.

Layering scenarios added:
  - Updating the LayeringPolicy with 2 layers in the layerOrder
    (down from 3) such that the site document should have its
    parent document recomputed as the global document.
  - A deletion action layering scenario (DH currently only has
    merge, replace scenarios in its funcitonal test suite.)

Documentation updated:
  - clarify the access levels for buckets, which has been a
    source of confusion.
  - update api-ref documentation for rendered-documents

Change-Id: Idb9b42351dfbdf75a19282c8478065e7564cfc26
This commit is contained in:
Felipe Monteiro 2017-12-20 18:05:05 +00:00
parent 18999390c7
commit 75d84312de
15 changed files with 256 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
deckhand/control/revision_documents.py
View File

@ -51,11 +51,8 @@ class RevisionDocumentsResource(api_base.BaseResource):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
order_by = sort_by = None
if 'order' in sanitized_params:
order_by = sanitized_params.pop('order')
if 'sort' in sanitized_params:
sort_by = sanitized_params.pop('sort')
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
filters = sanitized_params.copy()
filters['metadata.storagePolicy'] = ['cleartext']
@ -70,10 +67,11 @@ class RevisionDocumentsResource(api_base.BaseResource):
LOG.exception(six.text_type(e))
raise falcon.HTTPNotFound(description=e.format_message())
sorted_documents = utils.multisort(documents, sort_by, order_by)
# Sorts by creation date by default.
documents = utils.multisort(documents, sort_by, order_by)
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(sorted_documents)
resp.body = self.view_builder.list(documents)
class RenderedDocumentsResource(api_base.BaseResource):
@ -94,7 +92,8 @@ class RenderedDocumentsResource(api_base.BaseResource):
@policy.authorize('deckhand:list_cleartext_documents')
@common.sanitize_params([
'schema', 'metadata.name', 'metadata.label', 'status.bucket'])
'schema', 'metadata.name', 'metadata.label', 'status.bucket', 'order',
'sort'])
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
@ -122,15 +121,23 @@ class RenderedDocumentsResource(api_base.BaseResource):
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered.
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
user_filters = sanitized_params.copy()
user_filters['metadata.layeringDefinition.abstract'] = False
final_documents = [
rendered_documents = [
d for d in rendered_documents if utils.deepfilter(
d, **user_filters)]
if sort_by:
rendered_documents = utils.multisort(
rendered_documents, sort_by, order_by)
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(final_documents)
self._post_validate(final_documents)
resp.body = self.view_builder.list(rendered_documents)
self._post_validate(rendered_documents)
def _retrieve_documents_for_rendering(self, revision_id, **filters):
"""Retrieve all necessary documents needed for rendering. If a layering

12
deckhand/control/revisions.py
View File

@ -59,17 +59,15 @@ class RevisionsResource(api_base.BaseResource):
@policy.authorize('deckhand:list_revisions')
@common.sanitize_params(['tag', 'order', 'sort'])
def _list_revisions(self, req, resp, sanitized_params):
order_by = sort_by = None
if 'order' in sanitized_params:
order_by = sanitized_params.pop('order')
if 'sort' in sanitized_params:
sort_by = sanitized_params.pop('sort')
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
revisions = db_api.revision_get_all(**sanitized_params)
sorted_revisions = utils.multisort(revisions, sort_by, order_by)
if sort_by:
revisions = utils.multisort(revisions, sort_by, order_by)
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(sorted_revisions)
resp.body = self.view_builder.list(revisions)
@policy.authorize('deckhand:delete_revisions')
def on_delete(self, req, resp):

2
deckhand/engine/document.py
View File

@ -39,7 +39,7 @@ class Document(object):
"""
try:
return self._inner['metadata']['layeringDefinition']['abstract']
except KeyError:
except Exception:
return False
def get_schema(self):

5
deckhand/engine/layering.py
View File

@ -322,4 +322,7 @@ class DocumentLayering(object):
if 'children' in doc:
del doc['children']
return [d.to_dict() for d in self.layered_docs]
return (
[d.to_dict() for d in self.layered_docs] +
[self.layering_policy.to_dict()]
)

7
deckhand/engine/schema/v1_0/layering_policy_schema.py
View File

@ -30,6 +30,13 @@ schema = {
'storagePolicy': {
'type': 'string',
'enum': ['encrypted', 'cleartext']
},
'layeringDefinition': {
'type': 'object',
'properties': {
'abstract': {'type': 'boolean'}
},
'additionalProperties': False
}
},
'additionalProperties': False,

79
deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-layering.yaml
View File

@ -18,6 +18,24 @@ tests:
status: 204
response_headers: null
- name: add_bucket_layering
desc: |-
Create `layeringPolicy` in bucket layering with 3 layers: global, region
and site.
PUT: /api/v1.0/buckets/layering/documents
status: 200
data: |-
---
schema: deckhand/LayeringPolicy/v1
metadata:
schema: metadata/Control/v1
name: layering-policy
data:
layerOrder:
- global
- region
- site
- name: add_bucket_a
desc: Create documents for bucket a
PUT: /api/v1.0/buckets/a/documents
@ -33,13 +51,64 @@ tests:
- name: verify_layering
desc: Check for expected layering
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents
query_parameters:
sort:
- schema
- metadata.name
status: 200
response_multidoc_jsonpaths:
$.`len`: 1
$.[*].schema: example/Kind/v1
$.[*].metadata.name: site-1234
$.[*].metadata.schema: metadata/Document/v1
$.[*].data:
$.`len`: 3
$.[0].schema: deckhand/LayeringPolicy/v1
$.[1].schema: example/Kind/v1
$.[1].metadata.name: site-with-delete-action
$.[1].metadata.schema: metadata/Document/v1
$.[1].data: {}
$.[2].schema: example/Kind/v1
$.[2].metadata.name: site-with-merge-action
$.[2].metadata.schema: metadata/Document/v1
$.[2].data:
a:
z: 3
b: 4
- name: update_bucket_layering
desc: |-
Update `LayeringPolicy` in bucket 'layering', so that it only has 2
layers. This validates that, by dropping the middle layer "region",
layering is still performed using the global and site documents.
PUT: /api/v1.0/buckets/layering/documents
status: 200
data: |-
---
schema: deckhand/LayeringPolicy/v1
metadata:
schema: metadata/Control/v1
name: layering-policy
data:
layerOrder:
- global
- site
- name: verify_layering_again
desc: Check for expected layering with only global and site layers
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents
query_parameters:
sort:
- schema
- metadata.name
status: 200
response_multidoc_jsonpaths:
$.`len`: 3
$.[0].schema: deckhand/LayeringPolicy/v1
$.[1].schema: example/Kind/v1
$.[1].metadata.name: site-with-delete-action
$.[1].metadata.schema: metadata/Document/v1
$.[1].data: {}
$.[2].schema: example/Kind/v1
$.[2].metadata.name: site-with-merge-action
$.[2].metadata.schema: metadata/Document/v1
$.[2].data:
a:
x: 1
y: 2
b: 4

26
deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-layering.yaml
View File

@ -27,13 +27,16 @@ tests:
- name: verify_layering_2_layers
desc: Check for expected layering with 2 layers
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents
query_parameters:
sort: schema
status: 200
response_multidoc_jsonpaths:
$.`len`: 1
$.[*].schema: example/Kind/v1
$.[*].metadata.name: site-1234
$.[*].metadata.schema: metadata/Document/v1
$.[*].data:
$.`len`: 2
$.[0].schema: deckhand/LayeringPolicy/v1
$.[1].schema: example/Kind/v1
$.[1].metadata.name: site-1234
$.[1].metadata.schema: metadata/Document/v1
$.[1].data:
a:
x: 1
y: 2
@ -54,13 +57,16 @@ tests:
- name: verify_layering_3_layers
desc: Check for expected layering with 3 layers
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents
query_parameters:
sort: schema
status: 200
response_multidoc_jsonpaths:
$.`len`: 1
$.[*].schema: example/Kind/v1
$.[*].metadata.name: site-1234
$.[*].metadata.schema: metadata/Document/v1
$.[*].data:
$.`len`: 2
$.[0].schema: deckhand/LayeringPolicy/v1
$.[1].schema: example/Kind/v1
$.[1].metadata.name: site-1234
$.[1].metadata.schema: metadata/Document/v1
$.[1].data:
a:
z: 3
b: 4

10
deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-a.yaml
View File

@ -1,14 +1,4 @@
---
schema: deckhand/LayeringPolicy/v1
metadata:
schema: metadata/Control/v1
name: layering-policy
data:
layerOrder:
- global
- region
- site
---
schema: example/Kind/v1
metadata:
schema: metadata/Document/v1

16
deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-b.yaml
View File

@ -20,7 +20,7 @@ data:
schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: site-1234
name: site-with-merge-action
labels:
foo: bar
baz: qux
@ -33,4 +33,18 @@ metadata:
path: .
data:
b: 4
---
schema: example/Kind/v1
metadata:
schema: metadata/Document/v1
name: site-with-delete-action
layeringDefinition:
layer: site
parentSelector:
key1: value1
actions:
- method: delete
path: .a
# No data needed here, since we are deleting, not adding anything.
data: {}
...

64
deckhand/tests/unit/control/test_rendered_documents_controller.py
View File

@ -22,6 +22,7 @@ from deckhand import errors
from deckhand import factories
from deckhand.tests import test_utils
from deckhand.tests.unit.control import base as test_base
from deckhand import types
class TestRenderedDocumentsController(test_base.BaseControllerTest):
@ -54,6 +55,12 @@ class TestRenderedDocumentsController(test_base.BaseControllerTest):
self.assertEqual(200, resp.status_code)
rendered_documents = list(yaml.safe_load_all(resp.text))
# TODO(fmontei): Implement "negative" filter server-side.
rendered_documents = [
d for d in rendered_documents
if not d['schema'].startswith(types.LAYERING_POLICY_SCHEMA)
]
self.assertEqual(1, len(rendered_documents))
is_abstract = rendered_documents[0]['metadata']['layeringDefinition'][
'abstract']
@ -104,6 +111,12 @@ class TestRenderedDocumentsController(test_base.BaseControllerTest):
self.assertEqual(200, resp.status_code)
rendered_documents = list(yaml.safe_load_all(resp.text))
# TODO(fmontei): Implement "negative" filter server-side.
rendered_documents = [
d for d in rendered_documents
if not d['schema'].startswith(types.LAYERING_POLICY_SCHEMA)
]
self.assertEqual(1, len(rendered_documents))
self.assertEqual(new_name, rendered_documents[0]['metadata']['name'])
self.assertEqual(2, rendered_documents[0]['status']['revision'])
@ -231,6 +244,55 @@ class TestRenderedDocumentsControllerNegativeRBAC(
# Verify that the created document was not returned.
resp = self.app.simulate_get(
'/api/v1.0/revisions/%s/rendered-documents' % revision_id,
headers={'Content-Type': 'application/x-yaml'})
headers={'Content-Type': 'application/x-yaml'},
params={'schema': encrypted_document['schema']})
self.assertEqual(200, resp.status_code)
self.assertEmpty(list(yaml.safe_load_all(resp.text)))
class TestRenderedDocumentsControllerSorting(test_base.BaseControllerTest):
def test_rendered_documents_sorting_metadata_name(self):
rules = {'deckhand:list_cleartext_documents': '@',
'deckhand:list_encrypted_documents': '@',
'deckhand:create_cleartext_documents': '@'}
self.policy.set_rules(rules)
documents_factory = factories.DocumentFactory(2, [1, 1])
documents = documents_factory.gen_test({}, global_abstract=False,
region_abstract=False, site_abstract=False)
expected_names = ['bar', 'baz', 'foo']
for idx in range(len(documents)):
documents[idx]['metadata']['name'] = expected_names[idx]
resp = self.app.simulate_put(
'/api/v1.0/buckets/mop/documents',
headers={'Content-Type': 'application/x-yaml'},
body=yaml.safe_dump_all(documents))
self.assertEqual(200, resp.status_code)
revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][
'revision']
# Test ascending order.
resp = self.app.simulate_get(
'/api/v1.0/revisions/%s/rendered-documents' % revision_id,
params={'sort': 'metadata.name'}, params_csv=False,
headers={'Content-Type': 'application/x-yaml'})
self.assertEqual(200, resp.status_code)
retrieved_documents = list(yaml.safe_load_all(resp.text))
self.assertEqual(3, len(retrieved_documents))
self.assertEqual(expected_names,
[d['metadata']['name'] for d in retrieved_documents])
# Test descending order.
resp = self.app.simulate_get(
'/api/v1.0/revisions/%s/rendered-documents' % revision_id,
params={'sort': 'metadata.name', 'order': 'desc'},
params_csv=False, headers={'Content-Type': 'application/x-yaml'})
self.assertEqual(200, resp.status_code)
retrieved_documents = list(yaml.safe_load_all(resp.text))
self.assertEqual(3, len(retrieved_documents))
self.assertEqual(list(reversed(expected_names)),
[d['metadata']['name'] for d in retrieved_documents])

4
deckhand/tests/unit/engine/test_document_layering.py
View File

@ -16,6 +16,7 @@ from deckhand.engine import layering
from deckhand import errors
from deckhand import factories
from deckhand.tests.unit import base as test_base
from deckhand import types
class TestDocumentLayering(test_base.DeckhandTestCase):
@ -43,6 +44,9 @@ class TestDocumentLayering(test_base.DeckhandTestCase):
# should have a metadata.layeringDefinitionn.layer section.
rendered_documents = document_layering.render()
for doc in rendered_documents:
# No need to validate the LayeringPolicy: it remains unchanged.
if doc['schema'].startswith(types.LAYERING_POLICY_SCHEMA):
continue
layer = doc['metadata']['layeringDefinition']['layer']
if layer == 'site':
site_docs.append(doc)

4
doc/source/api_ref.rst
View File

@ -98,7 +98,9 @@ Valid query parameters are the same as for
``/revisions/{revision_id}/documents``, minus the paremters in
``metadata.layeringDetinition``, which are not supported.
Raises a 500 Internal Server Error if rendered documents fail schema
Raises a ``409 Conflict`` if a ``layeringPolicy`` document could not be found.
Raises a ``500 Internal Server Error`` if rendered documents fail schema
validation.
GET ``/revisions``

12
doc/source/overview.rst
View File

@ -92,6 +92,18 @@ However, documents can be read across different buckets and used together to
render finalized configuration documents, to be consumed by other services like
Armada, Drydock, Promenade or Shipyard.
In other words:
* Documents can be **read** from any bucket.
This is useful so that documents from different buckets can be used together
for layering and substitution.
* Documents can only be **written** to by the bucket that owns them.
This is useful because it offers the concept of ownership to a document in
which only the bucket that owns the document can manage it.
.. todo::
Deckhand should offer RBAC (Role-Based Access Control) around buckets. This

33
doc/source/testing.rst
View File

@ -27,6 +27,11 @@ Prerequisites
postgresql database for unit tests. The DB URL is set up as an environment
variable via ``PIFPAF_URL`` which is referenced by Deckhand's unit test suite.
When running `pifpaf run postgresql` (implicitly called by unit tests below),
pifpaf uses `pg_config` which can be installed on Ubuntu via::
sudo apt-get install libpq-dev -y
Guide
-----
@ -63,9 +68,18 @@ Functional testing
Prerequisites
-------------
Deckhand requires Docker to run its functional tests. A basic installation
guide for Docker for Ubuntu can be found
`here <https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/>`_.
* Docker
Deckhand requires Docker to run its functional tests. A basic installation
guide for Docker for Ubuntu can be found
`here <https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/>`_
* uwsgi
Can be installed on Ubuntu systems via::
sudo apt-get install uwsgi -y
Overview
--------
@ -91,10 +105,15 @@ The command executes ``tools/functional-tests.sh`` which:
6) An HTML report that visualizes the result of the test run is output to
``results/index.html``.
At this time, there are no functional tests for policy enforcement
verification. Negative tests will be added at a later date to confirm that
a 403 Forbidden is raised for each endpoint that does policy enforcement
absent necessary permissions.
Note that functional tests can be run concurrently; the flags ``--workers``
and ``--threads`` which are passed to ``uwsgi`` can be > 1.
.. todo::
At this time, there are no functional tests for policy enforcement
verification. Negative tests will be added at a later date to confirm that
a 403 Forbidden is raised for each endpoint that does policy enforcement
absent necessary permissions.
CICD
----

16
tools/functional-tests.sh
View File

@ -197,14 +197,14 @@ if [ -z "$DECKHAND_IMAGE" ]; then
# Set --workers 2, so that concurrency is always tested.
uwsgi \
--http :9000 \
-w deckhand.cmd \
--callable deckhand_callable \
--enable-threads \
--workers 2 \
--threads 1 \
-L \
--pyargv "--config-file $CONF_DIR/deckhand.conf" &> $STDOUT &
--http :9000 \
-w deckhand.cmd \
--callable deckhand_callable \
--enable-threads \
--workers 2 \
--threads 1 \
-L \
--pyargv "--config-file $CONF_DIR/deckhand.conf" &
else
log_section "Running Deckhand via Docker"
sudo docker run \