From eeee591f8efb8896e2297f2e04b71836a04cf49d Mon Sep 17 00:00:00 2001
From: Mark Burnett <mark.m.burnett@gmail.com>
Date: Wed, 20 Dec 2017 13:08:03 -0600
Subject: [PATCH] Add deckhand design_ref support
* Add ability to fetch design from Deckhand
* Add functional testing for Deckhand design_ref integration
* Update complete example to work with changes to Ceph chart
Change-Id: Ice25a27b340e68a8ab38a23021cd91e032ca537b
---
charts/promenade/templates/configmap-etc.yaml | 30 ++++
.../promenade/templates/deployment-api.yaml | 4 +
charts/promenade/values.yaml | 7 +
examples/basic/LayeringPolicy.yaml | 11 ++
examples/basic/armada-resources.yaml | 15 ++
examples/complete/Docker.yaml | 1 +
examples/complete/Genesis.yaml | 1 +
examples/complete/HostSystem.yaml | 1 +
examples/complete/Kubelet.yaml | 1 +
examples/complete/KubernetesNetwork.yaml | 1 +
examples/complete/LayeringPolicy.yaml | 11 ++
examples/complete/armada-resources.yaml | 161 +++++++++++-------
promenade/config.py | 7 +-
promenade/design_ref.py | 42 +++++
promenade/exceptions.py | 3 +-
promenade/options.py | 30 +++-
promenade/pki.py | 3 +-
promenade/promenade.py | 13 +-
requirements-direct.txt | 1 +
tests/unit/api/test_health_api.py | 4 +-
tests/unit/api/test_versions.py | 4 +-
tools/g2/lib/openstack.sh | 4 +
tools/g2/lib/registry.sh | 2 +-
tools/g2/manifests/conformance.json | 3 +-
tools/g2/manifests/genesis.json | 3 +-
tools/g2/manifests/integration.json | 11 +-
tools/g2/manifests/resiliency.json | 3 +-
tools/g2/manifests/smoke.json | 3 +-
tools/g2/stages/build-scripts.sh | 1 +
tools/g2/stages/generate-certificates.sh | 3 +-
tools/g2/stages/join-nodes.sh | 13 +-
tools/g2/stages/load-site-config.sh | 38 +++++
.../gate/config-templates/LayeringPolicy.yaml | 11 ++
tox.ini | 1 +
34 files changed, 352 insertions(+), 95 deletions(-)
create mode 100644 examples/basic/LayeringPolicy.yaml
create mode 100644 examples/complete/LayeringPolicy.yaml
create mode 100644 promenade/design_ref.py
create mode 100755 tools/g2/stages/load-site-config.sh
create mode 100644 tools/gate/config-templates/LayeringPolicy.yaml
diff --git a/charts/promenade/templates/configmap-etc.yaml b/charts/promenade/templates/configmap-etc.yaml
index 13d78be0..27411764 100644
--- a/charts/promenade/templates/configmap-etc.yaml
+++ b/charts/promenade/templates/configmap-etc.yaml
@@ -17,6 +17,34 @@ limitations under the License.
{{- if .Values.manifests.configmap_etc }}
{{- $envAll := . }}
+
+{{- if empty .Values.conf.promenade.keystone_authtoken.auth_uri -}}
+{{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.promenade.keystone_authtoken "auth_uri" | quote | trunc 0 -}}
+{{- end -}}
+
+# FIXME(sh8121att) fix for broken keystonemiddleware oslo config gen in newton - will remove in future
+{{- if empty .Values.conf.promenade.keystone_authtoken.auth_url -}}
+{{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup"| set .Values.conf.promenade.keystone_authtoken "auth_url" | quote | trunc 0 -}}
+{{- end -}}
+
+{{- $userIdentity := .Values.endpoints.identity.auth.user -}}
+
+{{- if empty .Values.conf.promenade.keystone_authtoken.project_name -}}
+{{- set .Values.conf.promenade.keystone_authtoken "project_name" $userIdentity.project_name | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.promenade.keystone_authtoken.project_domain_name -}}
+{{- set .Values.conf.promenade.keystone_authtoken "project_domain_name" $userIdentity.project_domain_name | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.promenade.keystone_authtoken.user_domain_name -}}
+{{- set .Values.conf.promenade.keystone_authtoken "user_domain_name" $userIdentity.user_domain_name | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.promenade.keystone_authtoken.username -}}
+{{- set .Values.conf.promenade.keystone_authtoken "username" $userIdentity.username | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.promenade.keystone_authtoken.password -}}
+{{- set .Values.conf.promenade.keystone_authtoken "password" $userIdentity.password | quote | trunc 0 -}}
+{{- end -}}
+
---
apiVersion: v1
kind: ConfigMap
@@ -25,4 +53,6 @@ metadata:
data:
api-paste.ini: |+
{{ include "helm-toolkit.utils.to_ini" .Values.conf.paste | indent 4 }}
+ promenade.conf: |+
+{{ include "helm-toolkit.utils.to_ini" .Values.conf.promenade | indent 4 }}
{{- end }}
diff --git a/charts/promenade/templates/deployment-api.yaml b/charts/promenade/templates/deployment-api.yaml
index c5d6a750..a1b2e63d 100644
--- a/charts/promenade/templates/deployment-api.yaml
+++ b/charts/promenade/templates/deployment-api.yaml
@@ -61,6 +61,10 @@ spec:
mountPath: /etc/promenade/api-paste.ini
subPath: api-paste.ini
readOnly: true
+ - name: promenade-etc
+ mountPath: /etc/promenade/promenade.conf
+ subPath: promenade.conf
+ readOnly: true
volumes:
- name: promenade-etc
configMap:
diff --git a/charts/promenade/values.yaml b/charts/promenade/values.yaml
index 653c8c5f..7764ff74 100644
--- a/charts/promenade/values.yaml
+++ b/charts/promenade/values.yaml
@@ -13,6 +13,12 @@
# limitations under the License.
conf:
+ promenade:
+ keystone_authtoken:
+ delay_auth_decision: true
+ auth_type: password
+ auth_section: keystone_authtoken
+
paste:
pipeline:main:
pipeline: authtoken promenade-api
@@ -22,6 +28,7 @@ conf:
forged_roles: admin
paste.filter_factory: promenade.control.middleware:noauth_filter_factory
app:promenade-api:
+ disable: ""
paste.app_factory: promenade.promenade:paste_start_promenade
images:
diff --git a/examples/basic/LayeringPolicy.yaml b/examples/basic/LayeringPolicy.yaml
new file mode 100644
index 00000000..46ae0c58
--- /dev/null
+++ b/examples/basic/LayeringPolicy.yaml
@@ -0,0 +1,11 @@
+---
+schema: deckhand/LayeringPolicy/v1
+metadata:
+ schema: metadata/Control/v1
+ name: layering-policy
+data:
+ layerOrder:
+ - global
+ - type
+ - site
+...
diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml
index c76ea09d..d644a3be 100644
--- a/examples/basic/armada-resources.yaml
+++ b/examples/basic/armada-resources.yaml
@@ -6,6 +6,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
release_prefix: ucp
chart_groups:
@@ -22,6 +23,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: Kubernetes proxy
sequenced: true
@@ -35,6 +37,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: Container networking via Calico
sequenced: true
@@ -49,6 +52,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: Cluster DNS
chart_group:
@@ -61,6 +65,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: Kubernetes components
chart_group:
@@ -76,6 +81,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: UCP platform components
chart_group:
@@ -85,6 +91,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: helm-toolkit
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: helm-toolkit
release: helm-toolkit
@@ -551,6 +561,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -651,6 +662,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -722,6 +734,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -783,6 +796,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -1022,6 +1036,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
chart_name: promenade
release: promenade
diff --git a/examples/complete/Docker.yaml b/examples/complete/Docker.yaml
index 9b303fad..5d261396 100644
--- a/examples/complete/Docker.yaml
+++ b/examples/complete/Docker.yaml
@@ -6,6 +6,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
config:
insecure-registries:
diff --git a/examples/complete/Genesis.yaml b/examples/complete/Genesis.yaml
index 70899c1a..8a5f5a1c 100644
--- a/examples/complete/Genesis.yaml
+++ b/examples/complete/Genesis.yaml
@@ -6,6 +6,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
hostname: n0
ip: 192.168.77.10
diff --git a/examples/complete/HostSystem.yaml b/examples/complete/HostSystem.yaml
index 6f4eebd7..d3f7c2b0 100644
--- a/examples/complete/HostSystem.yaml
+++ b/examples/complete/HostSystem.yaml
@@ -6,6 +6,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
files:
- path: /opt/kubernetes/bin/kubelet
diff --git a/examples/complete/Kubelet.yaml b/examples/complete/Kubelet.yaml
index 50016033..858cd4ff 100644
--- a/examples/complete/Kubelet.yaml
+++ b/examples/complete/Kubelet.yaml
@@ -6,6 +6,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
arguments:
- --cni-bin-dir=/opt/cni/bin
diff --git a/examples/complete/KubernetesNetwork.yaml b/examples/complete/KubernetesNetwork.yaml
index b5755010..9c3d0373 100644
--- a/examples/complete/KubernetesNetwork.yaml
+++ b/examples/complete/KubernetesNetwork.yaml
@@ -6,6 +6,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
dns:
cluster_domain: cluster.local
diff --git a/examples/complete/LayeringPolicy.yaml b/examples/complete/LayeringPolicy.yaml
new file mode 100644
index 00000000..46ae0c58
--- /dev/null
+++ b/examples/complete/LayeringPolicy.yaml
@@ -0,0 +1,11 @@
+---
+schema: deckhand/LayeringPolicy/v1
+metadata:
+ schema: metadata/Control/v1
+ name: layering-policy
+data:
+ layerOrder:
+ - global
+ - type
+ - site
+...
diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml
index 0654d6a8..de8b9600 100644
--- a/examples/complete/armada-resources.yaml
+++ b/examples/complete/armada-resources.yaml
@@ -6,6 +6,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
release_prefix: ucp
chart_groups:
@@ -24,6 +25,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: Kubernetes proxy
sequenced: true
@@ -37,6 +39,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: Container networking via Calico
sequenced: true
@@ -51,6 +54,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: Cluster DNS
chart_group:
@@ -63,6 +67,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: Ceph Storage
sequenced: true
@@ -77,6 +82,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: Kubernetes components
chart_group:
@@ -92,6 +98,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: UCP Infrastructure
sequenced: false
@@ -101,7 +108,6 @@ data:
- ucp-keystone
- maas-postgresql
- maas
- - ucp-etcd-rabbitmq
- ucp-rabbitmq
- ucp-barbican
- ingress
@@ -114,6 +120,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
description: UCP platform components
chart_group:
@@ -127,6 +134,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: helm-toolkit
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: helm-toolkit
release: helm-toolkit
@@ -151,6 +162,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -207,6 +219,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -451,6 +464,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -514,6 +528,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -593,6 +608,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -693,6 +709,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -764,6 +781,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -825,6 +843,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
substitutions:
-
src:
@@ -1061,6 +1080,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ceph
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: ceph
release: ceph
@@ -1069,17 +1092,11 @@ data:
wait:
timeout: 3600
install:
- no_hooks: false
+ no_hooks: true
upgrade:
no_hooks: false
pre:
delete:
- - name: ceph-bootstrap
- type: job
- labels:
- - application: ceph
- - component: bootstrap
- - release_group: armada-ucp
- name: ceph-mds-keyring-generator
type: job
labels:
@@ -1133,6 +1150,7 @@ data:
storage_secrets: true
ceph: true
rbd_provisioner: true
+ cephfs_provisioner: true
client_secrets: false
rgw_keystone_user_and_endpoints: false
bootstrap:
@@ -1148,6 +1166,7 @@ data:
ceph_config_helper: docker.io/port/ceph-config-helper:v1.7.5
ceph_daemon: docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04
ceph_rbd_provisioner: quay.io/external_storage/rbd-provisioner:v0.1.1
+ ceph_cephfs_provisioner: quay.io/external_storage/cephfs-provisioner:v0.1.1
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
ks_endpoints: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
ks_service: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
@@ -1164,6 +1183,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ucp-ceph-config
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: ucp-ceph-config
release: ucp-ceph-config
@@ -1213,6 +1236,7 @@ data:
deployment:
storage_secrets: false
ceph: false
+ cephfs_provisioner: false
rbd_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false
@@ -1228,6 +1252,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ucp-mariadb
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: ucp-mariadb
release: ucp-mariadb
@@ -1261,6 +1289,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ucp-memcached
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: ucp-memcached
release: ucp-memcached
@@ -1288,6 +1320,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ucp-keystone
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: ucp-keystone
release: keystone
@@ -1345,6 +1381,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: maas-postgresql
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: maas-postgresql
release: maas-postgresql
@@ -1381,6 +1421,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: maas
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: maas
release: maas
@@ -1392,15 +1436,13 @@ data:
values:
images:
tags:
+ bootstrap: sthussey/maas-region-controller:2.3
db_init: docker.io/postgres:9.5
- db_sync: quay.io/attcomdev/maas-region:latest
- dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
- maas_cache: docker.io/sthussey/maas-cache:cachetest
- maas_rack: quay.io/attcomdev/maas-rack:latest
- maas_region: quay.io/attcomdev/maas-region:latest
- bootstrap: quay.io/attcomdev/maas-region:latest
- export_api_key: quay.io/attcomdev/maas-region:latest
+ db_sync: sthussey/maas-region-controller:2.3
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
+ export_api_key: sthussey/maas-region-controller:2.3
+ maas_rack: sthussey/maas-rack-controller:2.3
+ maas_region: sthussey/maas-region-controller:2.3
labels:
rack:
node_selector_key: ucp-control-plane
@@ -1419,7 +1461,7 @@ data:
port: 31900
conf:
drydock:
- bootaction_url: http://${DRYDOCK_NODE_IP}:${DRYDOCK_NODE_PORT}/api/v1.0/bootactions/nodes/
+ bootaction_url: http://192.168.77.10:31000/api/v1.0/bootactions/nodes/
maas:
credentials:
secret:
@@ -1446,44 +1488,13 @@ data:
- helm-toolkit
---
schema: armada/Chart/v1
-metadata:
- schema: metadata/Document/v1
- name: ucp-etcd-rabbitmq
-data:
- chart_name: ucp-etcd-rabbitmq
- release: etcd-rabbitmq
- namespace: ucp
- install:
- no_hooks: false
- upgrade:
- no_hooks: false
- pre:
- delete: []
- post:
- delete: []
- create: []
- values:
- pod:
- replicas:
- etcd: 1
- labels:
- node_selector_key: ucp-control-plane
- node_selector_value: enabled
- images:
- tags:
- dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
- etcd: gcr.io/google_containers/etcd-amd64:2.2.5
- source:
- type: git
- location: https://git.openstack.org/openstack/openstack-helm
- subpath: etcd
- dependencies:
- - helm-toolkit
----
-schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ucp-rabbitmq
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: ucp-rabbitmq
release: rabbitmq
@@ -1500,7 +1511,7 @@ data:
values:
images:
tags:
- rabbitmq: quay.io/attcomdev/fuel-mcp-rabbitmq:ocata-unstable
+ rabbitmq: docker.io/rabbitmq:3.7
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
pod:
replicas:
@@ -1519,6 +1530,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ucp-barbican
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: ucp-barbican
release: barbican
@@ -1562,6 +1577,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ingress
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: ingress
release: ingress
@@ -1580,10 +1599,10 @@ data:
images:
tags:
entrypoint: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
- dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
- # https://github.com/kubernetes/ingress/blob/master/controllers/nginx/Changelog.md
- ingress: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.8
+ # https://github.com/kubernetes/ingress-nginx/blob/09524cd3363693463da5bf4a9bb3900da435ad05/Changelog.md#090
+ ingress: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0
error_pages: gcr.io/google_containers/defaultbackend:1.0
+ dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
source:
type: git
location: https://github.com/openstack/openstack-helm
@@ -1596,6 +1615,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: tiller
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: tiller
release: tiller
@@ -1623,6 +1646,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: deckhand
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: deckhand
release: deckhand
@@ -1664,6 +1691,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: drydock
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: drydock
release: drydock
@@ -1716,6 +1747,7 @@ metadata:
layeringDefinition:
abstract: false
layer: site
+ storagePolicy: cleartext
data:
chart_name: promenade
release: promenade
@@ -1724,17 +1756,6 @@ data:
wait:
timeout: 600
values:
- conf:
- paste:
- filter:authtoken:
- paste.filter_factory: 'keystonemiddleware.auth_token:filter_factory'
- admin_password: password
- admin_tenant_name: service
- admin_user: promenade
- delay_auth_decision: true
- identity_uri: 'http://keystone-api.ucp.svc.cluster.local/'
- service_token_roles_required: true
-
images:
tags:
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
@@ -1755,6 +1776,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: armada
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: armada
release: armada
@@ -1794,6 +1819,10 @@ schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: shipyard
+ layeringDefinition:
+ abstract: false
+ layer: site
+ storagePolicy: cleartext
data:
chart_name: shipyard
release: shipyard
diff --git a/promenade/config.py b/promenade/config.py
index 29680c75..25aee841 100644
--- a/promenade/config.py
+++ b/promenade/config.py
@@ -1,8 +1,8 @@
from . import exceptions, logging, validation
+from .design_ref import get_documents
import copy
import jinja2
import jsonpath_ng
-import requests
import yaml
__all__ = ['Configuration']
@@ -35,10 +35,7 @@ class Configuration:
@classmethod
def from_design_ref(cls, design_ref):
- response = requests.get(design_ref)
- response.raise_for_status()
-
- documents = list(yaml.safe_load_all(response.text))
+ documents = get_documents(design_ref)
validation.check_schemas(documents)
return cls(documents=documents)
diff --git a/promenade/design_ref.py b/promenade/design_ref.py
new file mode 100644
index 00000000..cbee1f68
--- /dev/null
+++ b/promenade/design_ref.py
@@ -0,0 +1,42 @@
+from . import logging
+from oslo_config import cfg
+import keystoneauth1.identity.v3
+import keystoneauth1.session
+import requests
+import yaml
+
+LOG = logging.getLogger(__name__)
+
+__all__ = ['get_documents']
+
+_DECKHAND_PREFIX = 'deckhand+'
+
+DH_TIMEOUT = 10 * 60 # 10 Minute timeout for fetching from Deckhand.
+
+
+def get_documents(design_ref):
+ LOG.debug('Fetching design_ref="%s"', design_ref)
+ if design_ref.startswith(_DECKHAND_PREFIX):
+ response = _get_from_deckhand(design_ref)
+ else:
+ response = _get_from_basic_web(design_ref)
+ LOG.debug('Got response for design_ref="%s"', design_ref)
+
+ response.raise_for_status()
+
+ return list(yaml.safe_load_all(response.text))
+
+
+def _get_from_basic_web(design_ref):
+ return requests.get(design_ref)
+
+
+def _get_from_deckhand(design_ref):
+ keystone_args = {}
+ for attr in ('auth_url', 'password', 'project_domain_name', 'project_name',
+ 'username', 'user_domain_name'):
+ keystone_args[attr] = cfg.CONF.get('keystone_authtoken', {}).get(attr)
+ auth = keystoneauth1.identity.v3.Password(**keystone_args)
+ session = keystoneauth1.session.Session(auth=auth)
+
+ return session.get(design_ref[len(_DECKHAND_PREFIX):], timeout=DH_TIMEOUT)
diff --git a/promenade/exceptions.py b/promenade/exceptions.py
index be2d40e2..5a2cd1e3 100644
--- a/promenade/exceptions.py
+++ b/promenade/exceptions.py
@@ -215,7 +215,7 @@ class PromenadeException(Exception):
if self.trace or debug:
LOG.exception(self.description)
else:
- LOG.error(self.description)
+ LOG.error(self.title + (self.description or ''))
class ApiError(PromenadeException):
@@ -238,6 +238,7 @@ class InvalidFormatError(PromenadeException):
class ValidationException(PromenadeException):
title = 'Validation Error'
+ status = falcon.HTTP_400
def massage_error_list(error_list, placeholder_description):
diff --git a/promenade/options.py b/promenade/options.py
index 4678eed7..9d528423 100644
--- a/promenade/options.py
+++ b/promenade/options.py
@@ -1,5 +1,33 @@
from oslo_config import cfg
+import keystoneauth1.loading
OPTIONS = []
-cfg.CONF.register_opts(OPTIONS)
+
+def setup(disable=None):
+ if disable is None:
+ disable = []
+ else:
+ disable = disable.split()
+
+ for name, func in GROUPS.items():
+ if name not in disable:
+ func()
+
+ cfg.CONF([], project='promenade')
+
+
+def register_application():
+ cfg.CONF.register_opts(OPTIONS)
+
+
+def register_keystone_auth():
+ cfg.CONF.register_opts(
+ keystoneauth1.loading.get_auth_plugin_conf_options('password'),
+ group='keystone_authtoken')
+
+
+GROUPS = {
+ 'promenade': register_application,
+ 'keystone': register_keystone_auth,
+}
diff --git a/promenade/pki.py b/promenade/pki.py
index e05e9e45..63866ebc 100644
--- a/promenade/pki.py
+++ b/promenade/pki.py
@@ -154,10 +154,11 @@ class PKI:
'metadata': {
'schema': 'metadata/Document/v1',
'name': name,
- 'layerinDefinition': {
+ 'layeringDefinition': {
'abstract': False,
'layer': 'site',
},
+ 'storagePolicy': 'cleartext',
},
'data': block_literal(data),
}
diff --git a/promenade/promenade.py b/promenade/promenade.py
index 558d8271..1fbebd17 100644
--- a/promenade/promenade.py
+++ b/promenade/promenade.py
@@ -11,15 +11,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from oslo_config import cfg
from promenade.control import api
-from promenade import options # noqa
+from promenade import options
from promenade import logging
from promenade import policy
-def start_promenade():
- cfg.CONF()
+def start_promenade(disable=""):
+ options.setup(disable=disable)
+
# Setup root logger
logging.setup(verbose=True)
@@ -33,7 +33,4 @@ def start_promenade():
# Initialization compatible with PasteDeploy
def paste_start_promenade(global_conf, **kwargs):
- return promenade
-
-
-promenade = start_promenade()
+ return start_promenade(**kwargs)
diff --git a/requirements-direct.txt b/requirements-direct.txt
index 4d90217a..2611b77d 100644
--- a/requirements-direct.txt
+++ b/requirements-direct.txt
@@ -3,6 +3,7 @@ falcon==1.2.0
jinja2==2.9.6
jsonpath-ng==1.4.3
jsonschema==2.6.0
+keystoneauth1==3.2.0
keystonemiddleware==4.17.0
kubernetes==3.0.0
oslo.context==2.19.2
diff --git a/tests/unit/api/test_health_api.py b/tests/unit/api/test_health_api.py
index 32b2bf1f..51f9be25 100644
--- a/tests/unit/api/test_health_api.py
+++ b/tests/unit/api/test_health_api.py
@@ -17,12 +17,12 @@ from falcon import testing
import pytest
from promenade.control import health_api
-from promenade.promenade import promenade
+from promenade import promenade
@pytest.fixture()
def client():
- return testing.TestClient(promenade)
+ return testing.TestClient(promenade.start_promenade(disable='keystone'))
def test_get_health(client):
diff --git a/tests/unit/api/test_versions.py b/tests/unit/api/test_versions.py
index 00d0c071..c1977058 100644
--- a/tests/unit/api/test_versions.py
+++ b/tests/unit/api/test_versions.py
@@ -17,12 +17,12 @@ from falcon import testing
import pytest
from promenade.control.api import VersionsResource
-from promenade.promenade import promenade
+from promenade import promenade
@pytest.fixture()
def client():
- return testing.TestClient(promenade)
+ return testing.TestClient(promenade.start_promenade(disable='keystone'))
def test_get_versions(client):
diff --git a/tools/g2/lib/openstack.sh b/tools/g2/lib/openstack.sh
index ba478e5d..d9dd70d5 100644
--- a/tools/g2/lib/openstack.sh
+++ b/tools/g2/lib/openstack.sh
@@ -26,6 +26,10 @@ EOBODY
rsync_cmd "${TEMP_DIR}/${REQUEST_BODY_PATH}" "${VIA}:/root/${REQUEST_BODY_PATH}"
ssh_cmd "${VIA}" curl -isS \
+ --fail \
+ --max-time 60 \
+ --retry 10 \
+ --retry-delay 15 \
-H 'Content-Type: application/json' \
-d "@/root/${REQUEST_BODY_PATH}" \
"${KEYSTONE_URL}/v3/auth/tokens" | grep 'X-Subject-Token' | awk '{print $2}' | sed "s;';;g" | sed "s;\r;;g"
diff --git a/tools/g2/lib/registry.sh b/tools/g2/lib/registry.sh
index 9aa51f44..e41b9a79 100644
--- a/tools/g2/lib/registry.sh
+++ b/tools/g2/lib/registry.sh
@@ -7,7 +7,7 @@ registry_down() {
}
registry_list_images() {
- FILES=($(find "$(config_configuration)" -type f -name '*.yaml'))
+ FILES=($(echo "$(config_configuration)" | xargs -n 1 -I DIRNAME find DIRNAME -type f -name '*.yaml'))
HOSTNAME_REGEX='[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}'
DOMAIN_NAME_REGEX="${HOSTNAME_REGEX}(\.${HOSTNAME_REGEX})*"
diff --git a/tools/g2/manifests/conformance.json b/tools/g2/manifests/conformance.json
index 5cca5d3a..02309f2a 100644
--- a/tools/g2/manifests/conformance.json
+++ b/tools/g2/manifests/conformance.json
@@ -1,6 +1,7 @@
{
"configuration": [
- "examples/basic"
+ "examples/basic",
+ "promenade/schemas"
],
"stages": [
{
diff --git a/tools/g2/manifests/genesis.json b/tools/g2/manifests/genesis.json
index 3dc48668..572f6506 100644
--- a/tools/g2/manifests/genesis.json
+++ b/tools/g2/manifests/genesis.json
@@ -1,6 +1,7 @@
{
"configuration": [
- "examples/complete"
+ "examples/complete",
+ "promenade/schemas"
],
"stages": [
{
diff --git a/tools/g2/manifests/integration.json b/tools/g2/manifests/integration.json
index 1adfb1b5..9b1139fa 100644
--- a/tools/g2/manifests/integration.json
+++ b/tools/g2/manifests/integration.json
@@ -1,6 +1,7 @@
{
"configuration": [
- "examples/complete"
+ "examples/complete",
+ "promenade/schemas"
],
"stages": [
{
@@ -27,12 +28,20 @@
"name": "Genesis",
"script": "genesis.sh"
},
+ {
+ "name": "Load Site Configuration",
+ "script": "load-site-config.sh",
+ "arguments": [
+ "-v", "n0"
+ ]
+ },
{
"name": "Join Master",
"script": "join-nodes.sh",
"arguments": [
"-v", "n0",
"-t",
+ "-d", "1",
"-n", "n1",
"-l", "calico-etcd=enabled",
"-l", "kubernetes-apiserver=enabled",
diff --git a/tools/g2/manifests/resiliency.json b/tools/g2/manifests/resiliency.json
index d8a0af85..fb108d30 100644
--- a/tools/g2/manifests/resiliency.json
+++ b/tools/g2/manifests/resiliency.json
@@ -1,6 +1,7 @@
{
"configuration": [
- "examples/basic"
+ "examples/basic",
+ "promenade/schemas"
],
"stages": [
{
diff --git a/tools/g2/manifests/smoke.json b/tools/g2/manifests/smoke.json
index db5c4b8c..1a9bb9a4 100644
--- a/tools/g2/manifests/smoke.json
+++ b/tools/g2/manifests/smoke.json
@@ -1,6 +1,7 @@
{
"configuration": [
- "examples/complete"
+ "examples/basic",
+ "promenade/schemas"
],
"stages": [
{
diff --git a/tools/g2/stages/build-scripts.sh b/tools/g2/stages/build-scripts.sh
index 8d3b7f49..97ca064c 100755
--- a/tools/g2/stages/build-scripts.sh
+++ b/tools/g2/stages/build-scripts.sh
@@ -20,4 +20,5 @@ docker run --rm -t \
-o scripts \
config/*.yaml
+mkdir -p "${TEMP_DIR}/nginx/"
cat "${TEMP_DIR}"/config/*.yaml > "${TEMP_DIR}/nginx/promenade.yaml"
diff --git a/tools/g2/stages/generate-certificates.sh b/tools/g2/stages/generate-certificates.sh
index 1a8b2d80..3ca95a25 100755
--- a/tools/g2/stages/generate-certificates.sh
+++ b/tools/g2/stages/generate-certificates.sh
@@ -7,10 +7,11 @@ source "${GATE_UTILS}"
OUTPUT_DIR="${TEMP_DIR}/config"
mkdir -p "${OUTPUT_DIR}"
chmod 777 "${OUTPUT_DIR}"
+OUTPUT_FILE="${OUTPUT_DIR}/combined.yaml"
for source_dir in $(config_configuration); do
log Copying configuration from "${source_dir}"
- cp "${WORKSPACE}/${source_dir}"/*.yaml "${OUTPUT_DIR}"
+ cat "${WORKSPACE}/${source_dir}"/*.yaml >> "${OUTPUT_FILE}"
done
registry_replace_references "${OUTPUT_DIR}"/*.yaml
diff --git a/tools/g2/stages/join-nodes.sh b/tools/g2/stages/join-nodes.sh
index 1eae7d00..551257e9 100755
--- a/tools/g2/stages/join-nodes.sh
+++ b/tools/g2/stages/join-nodes.sh
@@ -9,12 +9,17 @@ declare -a LABELS
declare -a NODES
GET_KEYSTONE_TOKEN=0
+USE_DECKHAND=0
-while getopts "e:l:n:tv:" opt; do
+while getopts "d:e:l:n:tv:" opt; do
case "${opt}" in
e)
ETCD_CLUSTERS+=("${OPTARG}")
;;
+ d)
+ USE_DECKHAND=1
+ DECKHAND_REVISION=${OPTARG}
+ ;;
l)
LABELS+=("${OPTARG}")
;;
@@ -58,7 +63,11 @@ render_curl_url() {
done
BASE_URL="${BASE_PROM_URL}/api/v1.0/join-scripts"
- DESIGN_REF="design_ref=http://192.168.77.1:7777/promenade.yaml"
+ if [[ ${USE_DECKHAND} == 1 ]]; then
+ DESIGN_REF="design_ref=deckhand%2Bhttp://deckhand-int.ucp.svc.cluster.local:9000/api/v1.0/revisions/${DECKHAND_REVISION}/rendered-documents"
+ else
+ DESIGN_REF="design_ref=http://192.168.77.1:7777/promenade.yaml"
+ fi
HOST_PARAMS="hostname=${NAME}&ip=$(config_vm_ip "${NAME}")"
echo "${BASE_URL}?${DESIGN_REF}&${HOST_PARAMS}${LABEL_PARAMS}"
diff --git a/tools/g2/stages/load-site-config.sh b/tools/g2/stages/load-site-config.sh
new file mode 100755
index 00000000..f3ab79e6
--- /dev/null
+++ b/tools/g2/stages/load-site-config.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -eu
+
+source "${GATE_UTILS}"
+
+while getopts "v:" opt; do
+ case "${opt}" in
+ v)
+ VIA=${OPTARG}
+ ;;
+ *)
+ echo "Unknown option"
+ exit 1
+ ;;
+ esac
+done
+shift $((OPTIND-1))
+
+if [ $# -gt 0 ]; then
+ echo "Unknown arguments specified: ${*}"
+ exit 1
+fi
+
+TOKEN=$(os_ks_get_token "${VIA}")
+
+DECKHAND_URL=http://deckhand-int.ucp.svc.cluster.local:9000/api/v1.0/buckets/prom/documents
+
+rsync_cmd "${TEMP_DIR}/nginx/promenade.yaml" "${VIA}:/root/promenade/promenade.yaml"
+ssh_cmd "${VIA}" curl -v \
+ --fail \
+ --max-time 300 \
+ --retry 10 \
+ --retry-delay 15 \
+ -H "X-Auth-Token: ${TOKEN}" \
+ -H "Content-Type: application/x-yaml" \
+ -T "/root/promenade/promenade.yaml" \
+ "${DECKHAND_URL}"
diff --git a/tools/gate/config-templates/LayeringPolicy.yaml b/tools/gate/config-templates/LayeringPolicy.yaml
new file mode 100644
index 00000000..46ae0c58
--- /dev/null
+++ b/tools/gate/config-templates/LayeringPolicy.yaml
@@ -0,0 +1,11 @@
+---
+schema: deckhand/LayeringPolicy/v1
+metadata:
+ schema: metadata/Control/v1
+ name: layering-policy
+data:
+ layerOrder:
+ - global
+ - type
+ - site
+...
diff --git a/tox.ini b/tox.ini
index 03ecf7e7..d04f616c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -29,6 +29,7 @@ commands =
yapf -ir {toxinidir}/promenade {toxinidir}/tests
[testenv:freeze]
+basepython=python3.5
deps = -r{toxinidir}/requirements-direct.txt
recreate = True
whitelist_externals = sh