update to focal and python 3.8

update dockerfile for python deckhand install
add deckhand version to chart 1.0
add chart version 0.2.0
update all packages to latest in requirements.txt
update zuul jobs for focal and python 3.8
remove zuul job functional-uwsgi-py38 in favor of functional-docker-py38
update tox config
typecast to string in re.sub() function
add stestr to test-requirements.txt
add SQLAlchemy jsonpickle sphinx-rtd-theme stestr to requirements.txt
deprecated function: BarbicanException -> BarbicanClientException
fix mock import using unittest
fix import collections to collections.abc
fix for collections modules for older than python 3.10 versions.
deprecated function: json -> to_json
deprecated function:  werkzeug.contrib.profiler ->
    werkzeug.middleware.profiler
deprecated function: falcon.AIP -> falcon.App
deprecation warning: switch from resp.body to resp.text
rename fixtures to dh_fixtures because there is an imported module
    fixtures
switch from stream.read to bounded_stream.read
deprecated function: falcon process_response needed additional parameter
deprecated function: falcon default_exception_handler changed parameter
    order
move from MagicMock object to falcon test generated object to fix
    incompatability with upgraded Falcon module.
Adjust gabbi tests to fix incompatability with upgraded DeepDiff module
update Makefile to execute ubuntu_focal
update HTK (helmtoolkit)
unpin barbican to pass integration tests
Use helm 3 in chart build.
    `helm serve` is removed in helm 3 so this moves
    to using local `file://` dependencies [0] instead.

Change-Id: I180416f480edea1b8968d80c993b3e1fcc95c08d

.dockerignore

@@ -1,2 +1,121 @@
-.tox
+# Byte-compiled / optimized / DLL files
-build
+**/__pycache__/
+*.py[cod]
+*$py.class
+
+# Unit tests
+src/bin/*/tests/unit/
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+**/env/
+**/build/
+**/develop-eggs/
+**/dist/
+**/downloads/
+**/eggs/
+**/.eggs/
+lib/
+**/lib64/
+parts/
+**/sdist/
+var/
+wheels/
+**/*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+**/htmlcov/
+cov/*
+**/.tox/
+**/.coverage
+**/.coverage.*
+**/.pytest_cache/
+**/.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+doc/_build/
+doc/*/_static/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mypy
+.mypy_cache/
+
+# Generated bogus docs
+ChangeLog
+AUTHORS
+
+# build/lint artifacts
+*.tgz
+**/*.tgz
+/charts/shipyard/charts
+/charts/shipyard/requirements.lock
+/charts/deps/*/
+.DS_Store
+
+# vscode
+.vscode/
+
+!src/lib/

.gitignore

@@ -111,6 +111,7 @@ ENV/
 # makefile build/lint artifacts
 /charts/deckhand/*.tgz
 /charts/deckhand/*.lock
+/charts/deps/*/
 # git
 Changelog
 AUTHORS

.zuul.yaml

@@ -14,52 +14,51 @@
     templates:
       - openstack-cover-jobs
       - docs-on-readthedocs
-      - openstack-python36-jobs
+      - openstack-python38-jobs
-      - openstack-python3-train-jobs
-      - openstack-python3-ussuri-jobs
     vars:
       rtd_webhook_id: '38572'
       rtd_project_name: 'airship-deckhand'
     check:
       jobs:
-        - deckhand-tox-py36-postgresql
+        - deckhand-tox-py38-postgresql
-        - deckhand-functional-uwsgi-py36
+        # Remove in favor of deckhand-functional-docker-py38 it runs the
-        - deckhand-functional-docker-py36-ubuntu_xenial
+        #  same tests just in a container
-        - deckhand-functional-docker-py36-ubuntu_bionic
+        # - deckhand-functional-uwsgi-py38
-        - deckhand-functional-docker-py36-opensuse
+        - deckhand-functional-docker-py38-ubuntu_focal
-        - deckhand-integration-uwsgi-py36
+        # non-voting, failing, incompatable with updates
-        - deckhand-integration-docker-py36-ubuntu_xenial
+        # - deckhand-functional-docker-py36-ubuntu_bionic
-        - deckhand-integration-docker-py36-ubuntu_bionic
+        # non-voting, failing, incompatable with updates
-        - deckhand-integration-docker-py36-opensuse
+        # Remove in favor of deckhand-uwsgi-docker-py38 it runs the
+        #  same tests just in a container
+        # - deckhand-integration-uwsgi-py38
+        - deckhand-integration-docker-py38-ubuntu_focal
+        # non-voting, failing, incompatable with updates
+        # - deckhand-integration-docker-py36-ubuntu_bionic
         - deckhand-chart-build-gate
         - deckhand-chart-build-latest-htk
-        - deckhand-docker-build-gate-ubuntu_xenial
+        - deckhand-docker-build-gate-ubuntu_focal
-        - deckhand-docker-build-gate-ubuntu_bionic
+        # non-voting, failing, incompatable with updates
-        - deckhand-docker-build-gate-opensuse
+        # - deckhand-docker-build-gate-ubuntu_bionic
+        - openstack-tox-pep8
         - deckhand-airskiff-deployment
     gate:
       jobs:
-        - deckhand-tox-py36-postgresql
+        - deckhand-tox-py38-postgresql
-        - deckhand-functional-docker-py36-ubuntu_xenial
+        - deckhand-functional-docker-py38-ubuntu_focal
         - deckhand-functional-docker-py36-ubuntu_bionic
-        - deckhand-functional-docker-py36-opensuse
+        - deckhand-integration-docker-py38-ubuntu_focal
-        - deckhand-integration-docker-py36-ubuntu_xenial
         - deckhand-integration-docker-py36-ubuntu_bionic
-        - deckhand-integration-docker-py36-opensuse
         - deckhand-chart-build-gate
-        - deckhand-docker-build-gate-ubuntu_xenial
+        - deckhand-docker-build-gate-ubuntu_focal
         - deckhand-docker-build-gate-ubuntu_bionic
-        - deckhand-docker-build-gate-opensuse
         - openstack-tox-pep8
     post:
       jobs:
         - deckhand-upload-git-mirror
-        - deckhand-docker-publish-ubuntu_xenial
+        - deckhand-docker-publish-ubuntu_focal
         - deckhand-docker-publish-ubuntu_bionic
-        - deckhand-docker-publish-opensuse
+        - deckhand-docker-tag-ubuntu_focal
-        - deckhand-docker-tag-ubuntu_xenial
         - deckhand-docker-tag-ubuntu_bionic
-        - deckhand-docker-tag-opensuse
 
 - nodeset:
     name: deckhand-single-node
@@ -73,22 +72,36 @@
       - name: primary
         label: ubuntu-bionic
 
+- nodeset:
+    name: deckhand-single-node-focal
+    nodes:
+      - name: primary
+        label: ubuntu-focal
+
+- nodeset:
+    name: deckhand-single-node-jammy
+    nodes:
+      - name: primary
+        label: ubuntu-jammy
+
 - job:
-    name: deckhand-tox-py36-postgresql
+    name: deckhand-tox-py38-postgresql
-    parent: openstack-tox-py36
+    parent: openstack-tox-py38
+    nodeset: deckhand-single-node-focal
     pre-run:
       - tools/gate/playbooks/install-postgresql.yaml
     vars:
-      tox_envlist: py36-postgresql
+      tox_envlist: py38-postgresql
 
 - job:
-    name: deckhand-functional-uwsgi-py36
+    name: deckhand-functional-uwsgi-py38
+    voting: false
     description: |
       Run tox-based functional tests for the Airship Deckhand project using a
       minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
-      for ephemeral PostgreSQL DB, under cPython version 3.6.
+      for ephemeral PostgreSQL DB, under cPython version 3.8.
     run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml
-    nodeset: deckhand-single-node
+    nodeset: deckhand-single-node-focal
     vars:
       tox_envlist: functional-dev
     irrelevant-files: &irrelevant-files
@@ -104,6 +117,7 @@
     description: |
       Base job for running deckhand functional tests. Runs tests
       against Docker image generated from source code.
+    nodeset: deckhand-single-node-focal
     roles:
       - zuul: openstack/openstack-helm-infra
     timeout: 3600
@@ -115,22 +129,22 @@
       - openstack/openstack-helm-infra
 
 - job:
-    name: deckhand-functional-docker-py36-ubuntu_xenial
+    name: deckhand-functional-docker-py38-ubuntu_focal
-    voting: false
     description: |
       Run tox-based functional tests for the Airship Deckhand project under
-      cPython version 3.6. Uses tox with the ``functional-py36`` environment.
+      cPython version 3.8. Uses tox with the ``functional-py38`` environment.
-      Ubuntu (xenial) image is built and used.
+      Ubuntu (focal) image is built and used.
     parent: deckhand-functional-docker-base
-    nodeset: deckhand-single-node
+    nodeset: deckhand-single-node-focal
     vars:
       tox_envlist: functional
       disable_keystone: true
-      distro: ubuntu_xenial
+      distro: ubuntu_focal
     irrelevant-files: *irrelevant-files
 
 - job:
     name: deckhand-functional-docker-py36-ubuntu_bionic
+    voting: false
     description: |
       Run tox-based functional tests for the Airship Deckhand project under
       cPython version 3.6. Uses tox with the ``functional-py36`` environment.
@@ -144,29 +158,15 @@
     irrelevant-files: *irrelevant-files
 
 - job:
-    name: deckhand-functional-docker-py36-opensuse
+    name: deckhand-integration-uwsgi-py38
     voting: false
-    description: |
-      Run tox-based functional tests for the Airship Deckhand project under
-      cPython version 3.6. Uses tox with the ``functional-py36`` environment.
-      Opensuse image is built and used.
-    parent: deckhand-functional-docker-base
-    nodeset: deckhand-single-node
-    vars:
-      tox_envlist: functional
-      disable_keystone: true
-      distro: opensuse_15
-    irrelevant-files: *irrelevant-files
-
-- job:
-    name: deckhand-integration-uwsgi-py36
     description: |
       Run tox-based integration tests for the Airship Deckhand project using a
       minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
-      for ephemeral PostgreSQL DB, under cPython version 3.6.
+      for ephemeral PostgreSQL DB, under cPython version 3.8.
     timeout: 3600
     run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml
-    nodeset: deckhand-single-node
+    nodeset: deckhand-single-node-focal
     irrelevant-files: *irrelevant-files
     vars:
       disable_keystone: true
@@ -177,7 +177,7 @@
       Build charts using pinned Helm toolkit.
     timeout: 900
     run: tools/gate/playbooks/build-charts.yaml
-    nodeset: deckhand-single-node
+    nodeset: deckhand-single-node-focal
 
 - job:
     name: deckhand-chart-build-latest-htk
@@ -185,7 +185,7 @@
       Build charts using latest Helm toolkit.
     timeout: 900
     run: tools/gate/playbooks/build-charts.yaml
-    nodeset: deckhand-single-node
+    nodeset: deckhand-single-node-focal
     vars:
       HTK_COMMIT: master
 
@@ -194,6 +194,7 @@
     description: |
       Base job for running deckhand integration tests. Runs tests
       against Docker image generated from source code.
+    nodeset: deckhand-single-node-focal
     timeout: 3600
     roles:
       - zuul: openstack/openstack-helm-infra
@@ -203,7 +204,8 @@
     pre-run:
       - tools/gate/playbooks/osh-infra-upgrade-host.yaml
       - tools/gate/playbooks/osh-infra-deploy-docker.yaml
-    run: tools/gate/playbooks/run-integration-tests-docker.yaml
+    run:
+      - tools/gate/playbooks/run-integration-tests-docker.yaml
     post-run: tools/gate/playbooks/osh-infra-collect-logs.yaml
     required-projects:
       - openstack/openstack-helm
@@ -214,16 +216,15 @@
       - ^releasenotes/.*$
 
 - job:
-    name: deckhand-integration-docker-py36-ubuntu_xenial
+    name: deckhand-integration-docker-py38-ubuntu_focal
-    voting: false
     description: |
       Run tox-based integration tests for the Airship Deckhand project under
-      cPython version 3.6. Builds ubuntu (xenial) deckhand image.
+      cPython version 3.8. Builds ubuntu (focal) deckhand image.
     parent: deckhand-integration-docker-base
     nodeset: openstack-helm-single-node
     vars:
       disable_keystone: false
-      distro: ubuntu_xenial
+      distro: ubuntu_focal
 
 - job:
     name: deckhand-integration-docker-py36-ubuntu_bionic
@@ -259,20 +260,7 @@
       - ^releasenotes/.*$
 
 - job:
-    name: deckhand-integration-docker-py36-opensuse
+    name: deckhand-docker-build-gate-ubuntu_focal
-    voting: false
-    description: |
-      Run tox-based integration tests for the Airship Deckhand project under
-      cPython version 3.6. Builds opensuse deckhand image.
-    parent: deckhand-integration-docker-base
-    nodeset: openstack-helm-single-node
-    vars:
-      disable_keystone: false
-      distro: opensuse_15
-
-- job:
-    name: deckhand-docker-build-gate-ubuntu_xenial
-    voting: false
     timeout: 1800
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: deckhand-single-node
@@ -285,13 +273,14 @@
       - ^setup.cfg$
     vars:
       publish: false
-      distro: ubuntu_xenial
+      distro: ubuntu_focal
       tags:
         dynamic:
           patch_set: true
 
 - job:
     name: deckhand-docker-build-gate-ubuntu_bionic
+    voting: false
     timeout: 1800
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: deckhand-single-node
@@ -304,35 +293,20 @@
           patch_set: true
 
 - job:
-    name: deckhand-docker-build-gate-opensuse
+    name: deckhand-docker-publish-ubuntu_focal
-    voting: false
-    timeout: 1800
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: deckhand-single-node
-    irrelevant-files: *non-code-files-template
-    vars:
-      publish: false
-      distro: opensuse_15
-      tags:
-        dynamic:
-          patch_set: true
-
-- job:
-    name: deckhand-docker-publish-ubuntu_xenial
-    voting: false
     description: |
       Runs on every merge, unless files in a dictionary below are changed.
       Builds and publishes container ubuntu images on quay.io with a set of tags
       listed in vars section. Waits in Zuul queue for a node (VM) assignment.
     timeout: 1800
     run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: deckhand-single-node
+    nodeset: deckhand-single-node-focal
     secrets:
       - airship_deckhand_quay_creds
     irrelevant-files: *non-code-files-template
     vars:
       publish: true
-      distro: ubuntu_xenial
+      distro: ubuntu_focal
       tags:
         dynamic:
           branch: true
@@ -342,6 +316,7 @@
 
 - job:
     name: deckhand-docker-publish-ubuntu_bionic
+    voting: false
     description: |
       Runs on every merge, unless files in a dictionary below are changed.
       Builds and publishes container ubuntu images on quay.io with a set of tags
@@ -363,31 +338,7 @@
           - latest
 
 - job:
-    name: deckhand-docker-publish-opensuse
+    name: deckhand-docker-tag-ubuntu_focal
-    voting: false
-    description: |
-      Runs on every merge, unless files in a dictionary below are changed.
-      Builds and publishes container opensuse images on quay.io with a set of tags
-      listed in vars section. Waits in Zuul queue for a node (VM) assignment.
-    timeout: 1800
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: deckhand-single-node
-    secrets:
-      - airship_deckhand_quay_creds
-    irrelevant-files: *non-code-files-template
-    vars:
-      publish: true
-      distro: opensuse_15
-      tags:
-        dynamic:
-          branch: true
-          commit: true
-        static:
-          - latest
-
-- job:
-    name: deckhand-docker-tag-ubuntu_xenial
-    voting: false
     description: |
       Runs on every merge when files in a dictionalry below are changed, and
       adds git commit id tag onto the ubuntu container image published on quay.io,
@@ -400,10 +351,11 @@
     secrets:
       - airship_deckhand_quay_creds
     vars:
-      distro: ubuntu_xenial
+      distro: ubuntu_focal
 
 - job:
     name: deckhand-docker-tag-ubuntu_bionic
+    voting: false
     description: |
       Runs on every merge when files in a dictionalry below are changed, and
       adds git commit id tag onto the ubuntu container image published on quay.io,
@@ -418,27 +370,6 @@
     vars:
       distro: ubuntu_bionic
 
-- job:
-    name: deckhand-docker-tag-opensuse
-    voting: false
-    description: |
-      Runs on every merge when files in a dictionalry below are changed, and
-      adds git commit id tag onto the opensuse container image published on quay.io,
-      which has `latest` tag set. Does not wait in queue for a node (VM)
-      assignment, runs almost immediately.
-    timeout: 1800
-    run: tools/gate/playbooks/docker-image-tag.yaml
-    nodeset:
-      nodes: []
-    secrets:
-      - airship_deckhand_quay_creds
-    vars:
-      distro: opensuse_15
-# file pattern here must be exactly the same as in
-# deckhand-docker-publish job above,
-# job will be executed on merge only when any of this files get changed
-    files: *non-code-files-template
-
 - secret:
     name: airship_deckhand_quay_creds
     data:

Makefile

@@ -25,7 +25,7 @@ USE_PROXY          ?= false
 PUSH_IMAGE         ?= false
 # use this variable for image labels added in internal build process
 LABEL              ?= org.airshipit.build=community
-DISTRO             ?= ubuntu_bionic
+DISTRO             ?= ubuntu_focal
 COMMIT             ?= $(shell git rev-parse HEAD)
 IMAGE              := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
 
@@ -37,13 +37,13 @@ images: build_deckhand
 
 # Create tgz of the chart
 .PHONY: charts
-charts: helm-init
+charts: helm-toolkit
 	$(HELM) dep up charts/deckhand
 	$(HELM) package charts/deckhand
 
 # Initialize local helm config
-.PHONY: helm-init
+.PHONY: helm-toolkit
-helm-init: helm-install
+helm-toolkit: helm-install
 	tools/helm_tk.sh $(HELM)
 
 # Install helm binary

charts/deckhand/Chart.yaml

@@ -15,7 +15,8 @@
 apiVersion: v1
 description: A Helm chart for Deckhand
 name: deckhand
-version: 0.1.0
+version: 0.2.0
+appVersion: 1.0
 keywords:
 - deckhand
 home: https://github.com/openstack/airship-deckhand

charts/deckhand/requirements.yaml

@@ -14,5 +14,5 @@
 
 dependencies:
   - name: helm-toolkit
-    repository: http://localhost:8879/charts
+    repository: file://../deps/helm-toolkit
     version: ">= 0.1.0"

deckhand/barbican/driver.py

@@ -163,7 +163,7 @@ class BarbicanDriver(object):
             raise errors.BarbicanServerException(details=str(e))
 
         payload = secret.payload
-        if secret.secret_type == 'opaque':
+        if secret.secret_type == 'opaque':  # nosec
             LOG.debug('Base64-decoding original payload '
                       'for document [%s, %s] %s.', *src_doc.meta)
             secret = self._base64_decode_payload(payload)
@@ -182,7 +182,7 @@ class BarbicanDriver(object):
         except (barbicanclient.exceptions.HTTPAuthError,
                 barbicanclient.exceptions.HTTPServerError) as e:
             LOG.exception(str(e))
-            raise errors.BarbicanException(details=str(e))
+            raise errors.BarbicanClientException(details=str(e))
         except barbicanclient.exceptions.HTTPClientError as e:
             if e.status_code == 404:
                 LOG.warning('Could not delete secret %s because it was not '

deckhand/common/document.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import collections
+from collections.abc import Iterable
 import re
 
 import hashlib
@@ -167,7 +167,7 @@ class DocumentDict(dict):
         :param documents: Documents to wrap in this class.
         :type documents: iterable
         """
-        if not isinstance(documents, collections.Iterable):
+        if not isinstance(documents, Iterable):
             documents = [documents]
 
         return [DocumentDict(d) for d in documents]

deckhand/common/utils.py

@@ -49,7 +49,7 @@ def to_camel_case(s):
 
 def to_snake_case(name):
     """Convert string to snake case."""
-    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
+    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', str(name))
     return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
 
 

deckhand/control/base.py

@@ -55,7 +55,7 @@ class BaseResource(object):
         :param allow_empty: Whether the request body can be empty.
         :returns: List of dicts if ``expect_list`` is True or else a dict.
         """
-        raw_data = req.stream.read(req.content_length or 0)
+        raw_data = req.bounded_stream.read()
 
         if not allow_empty and not raw_data:
             error_msg = ("The request body must not be empty.")

deckhand/control/buckets.py

@@ -64,7 +64,7 @@ class BucketsResource(api_base.BaseResource):
         created_documents = self._create_revision_documents(
             bucket_name, documents)
 
-        resp.body = self.view_builder.list(created_documents)
+        resp.text = self.view_builder.list(created_documents)
         resp.status = falcon.HTTP_200
 
     def _encrypt_secret_documents(self, documents):

deckhand/control/middleware.py

@@ -161,7 +161,7 @@ class YAMLTranslator(HookableMiddlewareMixin, object):
                 )
                 raise falcon.HTTPUnsupportedMediaType(description=message)
 
-    def process_response(self, req, resp, resource):
+    def process_response(self, req, resp, resource, req_succeeded):
         """Converts responses to ``application/x-yaml`` content type."""
         if resp.status != '204 No Content':
             resp.set_header('Content-Type', 'application/x-yaml')

deckhand/engine/revision_diff.py

@@ -248,7 +248,8 @@ def _diff_buckets(b1, b2):
                         # serializing to json then deserializing
                         # to dict.
                         data_changed = jsonpickle.decode(
-                            DeepDiff(d['data'], b2_tmp[k]['data']).json)
+                            DeepDiff(d['data'], b2_tmp[k]['data']).to_json())
+
                     # deepdiff doesn't provide custom exceptions;
                     # have to use Exception.
                     except Exception as ex:
@@ -256,7 +257,7 @@ def _diff_buckets(b1, b2):
                 try:
                     metadata_changed = jsonpickle.decode(
                         DeepDiff(d['metadata'],
-                                 b2_tmp[k]['metadata']).json)
+                                 b2_tmp[k]['metadata']).to_json())
                 except Exception as ex:
                     raise errors.DeepDiffException(details=str(ex))
 

deckhand/engine/utils.py

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import collections
+try:
+    from collections.abc import Mapping
+except ImportError:
+    from collections import Mapping
 
 from deckhand.common import utils
 
@@ -32,7 +35,7 @@ def deep_merge(dct, merge_dct):
     """
     for k, v in merge_dct.items():
         if (k in dct and isinstance(dct[k], dict) and
-                isinstance(merge_dct[k], collections.Mapping)):
+                isinstance(merge_dct[k], Mapping)):
             deep_merge(dct[k], merge_dct[k])
         else:
             dct[k] = merge_dct[k]

deckhand/errors.py

@@ -102,10 +102,10 @@ def format_error_resp(req,
     }
 
     resp.status = status_code
-    resp.body = yaml.safe_dump(error_response)
+    resp.text = yaml.safe_dump(error_response)
 
 
-def default_exception_handler(ex, req, resp, params):
+def default_exception_handler(req, resp, ex, params):
     """Catch-all exception handler for standardized output.
 
     If this is a standard falcon HTTPError, rethrow it for handling by

deckhand/service.py

@@ -17,7 +17,7 @@ import os
 import falcon
 from oslo_config import cfg
 from oslo_log import log
-from werkzeug.contrib.profiler import ProfilerMiddleware
+from werkzeug.middleware.profiler import ProfilerMiddleware
 
 from deckhand.control import base
 from deckhand.control import buckets
@@ -88,7 +88,7 @@ def deckhand_app_factory(global_config, **local_config):
                        middleware.ContextMiddleware(),
                        middleware.LoggingMiddleware()]
 
-    app = falcon.API(request_type=base.DeckhandRequest,
+    app = falcon.App(request_type=base.DeckhandRequest,
                      middleware=middleware_list)
     if CONF.profiler:
         LOG.warning("Profiler ENABLED. Expect significant "

deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml

@@ -241,7 +241,8 @@ tests:
       $.[0].'bucket_a diff'.document_changed.details:
         ('example/Kind/v1', 'doc-a'):
           data_changed:
-            dictionary_item_added: !!set {"root['foo']"}
+            dictionary_item_added:
+              ["root['foo']"]
           metadata_changed: {}
 
   - name: removing_key_in_doc-a
@@ -273,7 +274,8 @@ tests:
       $.[0].'bucket_a diff'.document_changed.details:
         ('example/Kind/v1', 'doc-a'):
           data_changed:
-            dictionary_item_removed: !!set {"root['foo']"}
+            dictionary_item_removed:
+              ["root['foo']"]
           metadata_changed: {}
 
   - name: verify_invalid_input

deckhand/tests/unit/barbican/test_cache.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
+from unittest import mock
 import testtools
 
 from deckhand.barbican import cache

deckhand/tests/unit/base.py

@@ -17,7 +17,7 @@ from __future__ import absolute_import
 import os
 
 import fixtures
-import mock
+from unittest import mock
 from oslo_config import cfg
 from oslo_log import log as logging
 import testtools
@@ -26,7 +26,7 @@ from deckhand.conf import config  # noqa: Calls register_opts(CONF)
 from deckhand.db.sqlalchemy import api as db_api
 from deckhand.engine import cache
 from deckhand.tests import test_utils
-from deckhand.tests.unit import fixtures as dh_fixtures
+from deckhand.tests.unit import dh_fixtures
 
 CONF = cfg.CONF
 logging.register_options(CONF)

deckhand/tests/unit/common/test_utils.py

@@ -14,7 +14,7 @@
 
 import hashlib
 import jsonpath_ng
-import mock
+from unittest import mock
 
 from oslo_serialization import jsonutils as json
 from testtools.matchers import Equals

deckhand/tests/unit/control/base.py

@@ -19,7 +19,7 @@ from falcon import testing as falcon_testing
 
 from deckhand import service
 from deckhand.tests.unit import base as test_base
-from deckhand.tests.unit import fixtures
+from deckhand.tests.unit import dh_fixtures
 
 
 class BaseControllerTest(test_base.DeckhandWithDBTestCase,
@@ -30,10 +30,10 @@ class BaseControllerTest(test_base.DeckhandWithDBTestCase,
         super(BaseControllerTest, self).setUp()
         self.app = falcon_testing.TestClient(
             service.deckhand_app_factory(None))
-        self.policy = self.useFixture(fixtures.RealPolicyFixture())
+        self.policy = self.useFixture(dh_fixtures.RealPolicyFixture())
         # NOTE: development_mode allows these unit tests to get around
         # Keystone authentication.
-        self.useFixture(fixtures.ConfPatcher(development_mode=True))
+        self.useFixture(dh_fixtures.ConfPatcher(development_mode=True))
 
     def _read_data(self, file_name):
         # Reads data from a file in the resources directory

deckhand/tests/unit/control/test_api_initialization.py

@@ -15,7 +15,7 @@
 import inspect
 import os
 
-import mock
+from unittest import mock
 
 from deckhand.common import utils
 from deckhand.control import api
@@ -72,7 +72,7 @@ class TestApi(test_base.DeckhandTestCase):
     @mock.patch('deckhand.service.falcon', autospec=True)
     def test_init_application(self, mock_falcon, mock_logging,
                               mock_db_api, _):
-        mock_falcon_api = mock_falcon.API.return_value
+        mock_falcon_api = mock_falcon.App.return_value
         self.override_config(
             'connection', mock.sentinel.db_connection, group='database')
 

deckhand/tests/unit/control/test_base_controller.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
+from unittest import mock
 
 from deckhand.control import base as api_base
 from deckhand.tests.unit.control import base as test_base

deckhand/tests/unit/control/test_buckets_controller.py

@@ -14,7 +14,7 @@
 
 import yaml
 
-import mock
+from unittest import mock
 from oslo_config import cfg
 
 from deckhand.engine import secrets_manager

deckhand/tests/unit/control/test_errors.py

@@ -16,7 +16,7 @@ import os
 import yaml
 
 import falcon
-import mock
+from unittest import mock
 
 from deckhand.common import document as document_wrapper
 from deckhand import policy
@@ -58,7 +58,7 @@ class TestErrorFormatting(test_base.BaseControllerTest):
             'message': 'Unhandled Exception raised: test error',
             'metadata': {}
         }
-        body = yaml.load(resp.text)
+        body = yaml.safe_load(resp.text)
 
         self.assertEqual(500, resp.status_code)
         self.assertEqual(expected, body)

deckhand/tests/unit/control/test_middleware.py

@@ -14,7 +14,7 @@
 
 import yaml
 
-import mock
+from unittest import mock
 
 from deckhand import factories
 from deckhand.tests.unit.control import base as test_base
@@ -118,16 +118,17 @@ class TestYAMLTranslatorNegative(test_base.BaseControllerTest):
                 'errorType': 'HTTPMissingHeader',
                 'messageList': [{
                     'error': True,
-                    'message': "The Content-Type header is required."
+                    'message': "The \"Content-Type\" header is required."
                 }]
             },
             'kind': 'Status',
-            'message': "The Content-Type header is required.",
+            'message': "The \"Content-Type\" header is required.",
             'metadata': {},
             'reason': 'Unspecified',
             'retry': False,
             'status': 'Failure'
         }
+
         self.assertEqual(expected, yaml.safe_load(resp.content))
 
     def test_request_with_invalid_content_type_raises_exception(self):

deckhand/tests/unit/control/test_rendered_documents_controller.py

@@ -15,7 +15,7 @@
 import re
 import yaml
 
-import mock
+from unittest import mock
 
 from deckhand.common.document import DocumentDict as dd
 from deckhand.control import revision_documents

deckhand/tests/unit/control/test_revision_documents_controller.py

@@ -14,7 +14,7 @@
 
 import yaml
 
-import mock
+from unittest import mock
 
 from deckhand.common.document import DocumentDict as document_dict
 from deckhand.engine import secrets_manager

deckhand/tests/unit/control/test_revisions_controller.py

@@ -14,7 +14,7 @@
 
 import os
 
-import mock
+from unittest import mock
 import yaml
 
 from deckhand.engine import secrets_manager

deckhand/tests/unit/control/test_revisions_rollback_controller.py

@@ -14,7 +14,7 @@
 
 import yaml
 
-import mock
+from unittest import mock
 
 from deckhand.engine import secrets_manager
 from deckhand import factories

deckhand/tests/unit/db/test_documents.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
+from unittest import mock
 import testtools
 
 from deckhand.db.sqlalchemy import api as db_api

deckhand/tests/unit/dh_fixtures.py

@@ -22,7 +22,7 @@ import os
 import yaml
 
 import fixtures
-import mock
+from unittest import mock
 from oslo_config import cfg
 from oslo_policy import opts as policy_opts
 from oslo_policy import policy as oslo_policy
@@ -41,7 +41,6 @@ class ConfPatcher(fixtures.Fixture):
     it's teardown.
 
     """
-
     def __init__(self, **kwargs):
         """Constructor
 

deckhand/tests/unit/engine/test_document_layering.py

@@ -14,7 +14,7 @@
 
 import yaml
 
-import mock
+from unittest import mock
 
 from deckhand.engine import layering
 from deckhand.engine import secrets_manager

deckhand/tests/unit/engine/test_document_layering_and_substitution.py

@@ -14,7 +14,7 @@
 
 import itertools
 
-import mock
+from unittest import mock
 import yaml
 
 from deckhand import factories

deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
+from unittest import mock
 
 from deckhand.engine import layering
 from deckhand import errors

deckhand/tests/unit/engine/test_document_layering_negative.py

@@ -14,7 +14,7 @@
 
 import copy
 
-import mock
+from unittest import mock
 
 from deckhand.engine import layering
 from deckhand import errors

deckhand/tests/unit/engine/test_document_validation.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
+from unittest import mock
 
 from deckhand.common import utils
 from deckhand.engine import document_validation

deckhand/tests/unit/engine/test_document_validation_negative.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
+from unittest import mock
 
 from deckhand.engine import document_validation
 from deckhand import errors

deckhand/tests/unit/engine/test_secrets_manager.py

@@ -15,7 +15,7 @@
 import copy
 import yaml
 
-import mock
+from unittest import mock
 from oslo_serialization import base64
 from oslo_utils import uuidutils
 import testtools

deckhand/tests/unit/test_policy.py

@@ -13,13 +13,14 @@
 # limitations under the License.
 
 import falcon
-import mock
+from falcon import testing as falcon_testing
+from unittest import mock
 from oslo_policy import policy as common_policy
 
 from deckhand.control import base as api_base
 import deckhand.policy
 from deckhand.tests.unit import base as test_base
-from deckhand.tests.unit import fixtures
+from deckhand.tests.unit import dh_fixtures
 
 
 class PolicyBaseTestCase(test_base.DeckhandTestCase):
@@ -34,7 +35,7 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase):
             "deckhand:list_cleartext_documents": [['rule:admin_api']]
         }
 
-        self.useFixture(fixtures.RealPolicyFixture(False))
+        self.useFixture(dh_fixtures.RealPolicyFixture(False))
         self._set_rules()
 
     def _set_rules(self):
@@ -51,9 +52,23 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase):
         noop(*api_args)
 
     def _get_args(self):
+        env = falcon_testing.create_environ(
+            path='/',
+            query_string='',
+            scheme='http',
+            host='falconframework.org',
+            port=None,
+            headers={'Content-Type': 'application/x-yaml'},
+            app='',
+            body='',
+            method='POST',
+            wsgierrors=None,
+            file_wrapper=None)
+
+        falcon_req = api_base.DeckhandRequest(env)
+
         # Returns the first two arguments that would be passed to any falcon
         # on_{HTTP_VERB} method: (self (which is mocked), falcon Request obj).
-        falcon_req = api_base.DeckhandRequest(mock.MagicMock())
         return (mock.Mock(), falcon_req)
 
 

doc/requirements.txt

@@ -4,6 +4,7 @@
 sphinx>=1.6.2 # BSD
 sphinx_rtd_theme
 reno>=2.5.0 # Apache-2.0
+pylibyaml==0.1.0
 plantuml
 sphinxcontrib-apidoc>=0.2.0  # BSD
 sphinxcontrib-plantuml

images/deckhand/Dockerfile.ubuntu_focal

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG FROM=ubuntu:16.04
+ARG FROM=ubuntu:20.04
 FROM ${FROM}
 
 LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
@@ -51,7 +51,7 @@ RUN set -x && \
     python3-pip \
     python3-setuptools \
     --no-install-recommends \
-    && python3 -m pip install -U 'pip<21.0' \
+    && python3 -m pip install -U pip \
     && apt-get clean \
     && rm -rf \
         /var/lib/apt/lists/* \
@@ -88,8 +88,16 @@ RUN chown -R deckhand: /home/deckhand \
 
 # Set work directory and install dependencies
 WORKDIR /home/deckhand
-RUN pip3 install -r requirements.txt
+RUN pip3 install --no-cache-dir -r requirements.txt
-RUN python3 setup.py install
+
+# Setting deckhand version for BPR
+ENV PBR_VERSION 1.0
+
+
+COPY . /opt/deckhand
+RUN pip3 install -e /opt/deckhand \
+    && echo "/opt/deckhand" \
+        > /usr/local/lib/python3.8/dist-packages/deckhand.pth
 
 # Set user to deckhand
 USER deckhand

requirements.txt

@@ -2,51 +2,54 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-hacking>=3.0.1,<3.1.0
+hacking==3.0.1
-
+SQLAlchemy==1.4.23
-alembic==1.0.1
+alembic==1.7.1
-amqp<2.7,>=2.6.0
+amqp==5.0.8
-beaker==1.10.0
+Beaker==1.12.0
-cryptography>=2.7
+cryptography==3.4.8
-deepdiff==3.3.0
+deepdiff==5.8.1
-falcon==1.4.1
+falcon==3.1.1
-jsonpath-ng==1.4.3
+jsonpath-ng==1.5.3
-jsonschema>=3.0.1,<4
+jsonschema==3.2.0
-keystoneauth1>=3.18.0
+keystoneauth1==5.1.1
-keystonemiddleware==5.3.0
+keystonemiddleware==10.2.0
-kombu<4.7,>=4.6.10
+kombu==5.1.0
-networkx==2.2
+networkx==2.6.2
-oslo.cache==1.38.1
+oslo.cache==2.8.2
-oslo.concurrency==3.28.1
+oslo.concurrency==4.4.1
-oslo.config==7.0.0
+oslo.config==8.7.1
-oslo.context>=2.21.0
+oslo.context==5.0.0
-oslo.messaging==9.1.1
+oslo.messaging==12.9.4
-oslo.db==4.41.1
+oslo.db==11.0.0
-oslo.log==3.45.2
+oslo.log==4.6.0
-oslo.middleware==3.36.0
+oslo.middleware==4.4.0
-oslo.policy==1.40.1
+oslo.policy==4.0.0
-oslo.serialization==2.29.2
+oslo.serialization==4.2.0
-oslo.utils==3.42.1
+oslo.utils==4.10.2
-pbr==5.4.5
+pbr==5.6.0
-PasteDeploy==1.5.2
+PasteDeploy==3.0.1
-Paste==3.0.1
+Paste==3.5.0
-psycopg2-binary==2.8.4
+psycopg2-binary==2.9.5
-pylibyaml~=0.1
+pylibyaml==0.1.0
-pyyaml~=5.1
+PyYAML==5.4.1
-python-dateutil>=2.8.1
+python-dateutil==2.8.2
 
 # TODO(alanmeadows)
 # this must match the container service
 # likely this should be imported from a
 # container sidecar long-term
-python-barbicanclient==4.7.0
+python-barbicanclient==5.2.0
 python-keystoneclient==3.22.0
 python-memcached==1.59
 
-Routes==2.4.1
+Routes==2.5.1
-six>=1.15.0
+six==1.16.0
-stevedore>=1.30.0
+sphinx-rtd-theme==1.1.1
-urllib3==1.25.9
+stestr==3.2.0
-uwsgi~=2.0.19.1
+stevedore==4.1.1
+urllib3==1.26.6
+uWSGI==2.0.21
 # To support profiling in non-prod
-Werkzeug==0.16.1
+Werkzeug==2.0.1
+jsonpickle

setup.cfg

@@ -1,10 +1,11 @@
 [metadata]
 name = Deckhand
+version = 1.0
 summary = Storage service for YAML-based configuration documents, which are managed through version control and automatically validated.
-description-file = README.rst
+description_file = README.rst
 author = The Airship Authors
-author-email = airship-discuss@lists.airshipit.org
+author_email = airship-discuss@lists.airshipit.org
-home-page = https://airship-deckhand.readthedocs.io/
+home_page = https://airship-deckhand.readthedocs.io/
 
 classifier =
     Intended Audience :: Information Technology
@@ -13,8 +14,8 @@ classifier =
     Operating System :: POSIX :: Linux
     Programming Language :: Python
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.8
-    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.10
 
 [files]
 packages =

setup.py

@@ -12,16 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import setuptools
+from setuptools import setup
 
-# In python < 2.7.4, a lazy loading of package `pbr` will break
+setup(
-# setuptools if some other modules registered functions in `atexit`.
-# solution from: http://bugs.python.org/issue15881#msg170215
-try:
-    import multiprocessing  # noqa
-except ImportError:
-    pass
-
-setuptools.setup(
     setup_requires=['pbr>=2.0.0'],
-    pbr=True)
+    pbr=True
+)

test-requirements.txt

@@ -2,22 +2,28 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-amqp<2.7,>=2.6.0
+amqp==5.0.8
-coverage==4.5.1
+coverage==5.5
 fixtures==3.0.0
-python-subunit>=1.4.0
+python-subunit==1.4.0
-os-testr==1.0.0
+os-testr==2.0.1
 testrepository==0.0.20
-testtools==2.3.0
+testtools==2.5.0
-bandit==1.6.2
+bandit==1.7.4
 # NOTE(felipemonteiro): Pin here because later versions require that
 # content-type be present in empty responses.
 gabbi==1.35.1
-pifpaf==2.1.2
+pifpaf==3.1.5
-oslotest==3.7.0
+oslotest==4.5.0
-yq>=2.7.2
+yq==3.1.0
-tox
+tox<=4
-pylibyaml~=0.1
+pylibyaml==0.1.0
-six>=1.15.0
+six==1.16.0
-pyparsing>=2.1.0
+pyparsing==2.4.7
-openstacksdk==0.36.5
+openstacksdk==0.59.0
+stestr==3.2.0
+requests==2.28.2
+urllib3==1.26.6
+chardet==4.0.0
+pyproject-api==1.5.0
+packaging>=23

tools/gate/playbooks/osh-infra-deploy-docker.yaml

@@ -20,9 +20,9 @@
   gather_facts: False
   become: yes
   roles:
-    - deploy-python
+    - ensure-python
   tags:
-    - deploy-python
+    - ensure-python
 
 - hosts: all
   vars_files:
@@ -31,15 +31,20 @@
     work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
   gather_facts: True
   become: yes
+
   roles:
-    - setup-firewall
+    - clear-firewall
+    - ensure-python
+    - ensure-pip
+    - ensure-docker
+    - ensure-tox
     - install-test-requirements
-    - deploy-python-pip
-    - deploy-docker
     - deploy-jq
   tags:
-    - setup-firewall
+    - clear-firewall
+    - ensure-python
+    - ensure-pip
+    - ensure-docker
+    - ensure-tox
     - install-test-requirements
-    - deploy-python-pip
-    - deploy-docker
     - deploy-jq

tools/gate/playbooks/osh-infra-upgrade-host.yaml

@@ -20,9 +20,9 @@
   gather_facts: False
   become: yes
   roles:
-    - deploy-python
+    - ensure-python
   tags:
-    - deploy-python
+    - ensure-python
 
 - hosts: all
   vars_files:

tools/gate/playbooks/vars.yaml

@@ -12,4 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-barbican_stable_commit: 3ac3caa0138d44aa6031848d5b83802128a169b5
+# zuul_airship_deckhand_relative_path: ../airship-deckhand
+# barbican_stable_commit: 3ac3caa0138d44aa6031848d5b83802128a169b5

tools/gate/roles/build-images/defaults/main.yaml

@@ -17,4 +17,4 @@ proxy:
   https: null
   noproxy: null
 
-distro: ubuntu_bionic
+distro: ubuntu_focal

tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml

@@ -36,6 +36,7 @@
               docker build \
                 --network host \
                 --force-rm \
+                --tag deckhand \
                 --label zuul \
                 --file images/deckhand/Dockerfile.{{ distro }} \
                 {% if zuul_site_mirror_fqdn is defined and zuul_site_mirror_fqdn %}
@@ -57,6 +58,7 @@
               docker build \
                 --network host \
                 --force-rm \
+                --tag deckhand \
                 --label zuul \
                 --file images/deckhand/Dockerfile.{{ distro }} \
                 --build-arg HTTP_PROXY="{{ proxy.http }}" \
@@ -81,4 +83,4 @@
       register: airship_deckhand_image_id
       args:
         chdir: "{{ airship_deckhand_path.stdout }}"
-      become: yes
+      become: yes

tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml

@@ -15,7 +15,7 @@
 - name: Deploy Barbican
   shell: |
     set -xe;
-    git reset --hard "{{ barbican_stable_commit | default('') }}"
+    # git reset --hard "{{ barbican_stable_commit | default('') }}"
     ./tools/deployment/component/barbican/barbican.sh
   args:
     chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}"

tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml

@@ -20,6 +20,7 @@
     docker run \
       --rm \
       --net=host \
+      --dns 10.96.0.10 \
       -v "{{ deckhand_conf_dir | default('') }}":/etc/deckhand "{{ airship_deckhand_image_id.stdout }}" alembic upgrade head
 
     # Allow migrations to complete.
@@ -28,10 +29,13 @@
 
 - name: Deploy Deckhand Container
   shell: |-
+    set -ex;
+
     docker run \
       --rm \
       --net=host \
       -p 9000:9000 \
+      --dns 10.96.0.10 \
       -v "{{ deckhand_conf_dir | default('') }}":/etc/deckhand "{{ airship_deckhand_image_id.stdout }}" server &
 
     # Give the server a chance to come up. Better to poll a health check.

tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml

@@ -61,6 +61,6 @@
   environment:
     DECKHAND_TEST_URL: "127.0.0.1:9000"
     DECKHAND_TEST_DIR: "{{ airship_deckhand_path.stdout }}/deckhand/tests/integration/gabbits"
-    BARBICAN_STABLE_COMMIT: "{{ barbican_stable_commit }}"
+    # BARBICAN_STABLE_COMMIT: "{{ barbican_stable_commit }}"
   register: result
   failed_when: "result.rc != 0"

tools/gate/scripts/020-deploy-postgresql.sh

@@ -23,7 +23,7 @@ POSTGRES_ID=$(
         -e POSTGRES_DB=deckhand \
         -e POSTGRES_USER=deckhand \
         -e POSTGRES_PASSWORD=password \
-            postgres:9.5
+            postgres:14.6
 )
 
 POSTGRES_IP=$(

tools/helm_install.sh

@@ -17,7 +17,7 @@
 set -x
 
 HELM=$1
-HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v2.17.0-linux-amd64.tar.gz"}
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz"}
 
 
 function install_helm_binary {

tools/helm_tk.sh

@@ -12,58 +12,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
-# Script to setup helm-toolkit and helm dep up the deckhand chart
+
-#
+set -eux
-HELM=$1
+
-HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"}
+HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
-HTK_PATH=${HTK_PATH:-""}
 HTK_STABLE_COMMIT=${HTK_COMMIT:-"f4972121bcb41c8d74748917804d2b239ab757f9"}
-BUILD_DIR=${BUILD_DIR:-$(mktemp -d)}
 
-if [[ ! -z $(echo $http_proxy) ]]
+TMP_DIR=$(mktemp -d)
-then
-  export no_proxy=$no_proxy,127.0.0.1
-fi
 
-set -x
+{
-
+    HTK_REPO_DIR=$TMP_DIR/htk
-function helm_serve {
+    git clone "$HTK_REPO" "$HTK_REPO_DIR"
-  if [[ -d "$HOME/.helm" ]]; then
+    (cd "$HTK_REPO_DIR" && git reset --hard "${HTK_STABLE_COMMIT}")
-     echo ".helm directory found"
+    cp -r "${HTK_REPO_DIR}/helm-toolkit" charts/deps/
-  else
-     ${HELM} init --client-only --skip-refresh
-  fi
-  if [[ -z $(curl -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; then
-     ${HELM} serve & > /dev/null
-     while [[ -z $(curl -s 127.0.0.1:8879 | grep 'Helm Repository') ]]; do
-        sleep 1
-        echo "Waiting for Helm Repository"
-     done
-  else
-     echo "Helm serve already running"
-  fi
-
-  if ${HELM} repo list | grep -q "^stable" ; then
-     ${HELM} repo remove stable
-  fi
-
-  ${HELM} repo add local http://localhost:8879/charts
 }
 
-mkdir -p "$BUILD_DIR"
+rm -rf "${TMP_DIR}"
-pushd "$BUILD_DIR"
-git clone $HTK_REPO || true
-pushd openstack-helm-infra/$HTK_PATH
-git reset --hard "${HTK_STABLE_COMMIT}"
-
-helm_serve
-# OSH Makefile is bugged, so ensure helm is in the path
-if [[ ${HELM} != "helm" ]]
-then
-  export PATH=${PATH}:$(dirname ${HELM})
-fi
-
-make helm-toolkit
-popd && popd
-${HELM} dep up charts/deckhand

tools/integration-tests.sh

@@ -47,7 +47,7 @@ function deploy_osh_keystone_barbican {
     fi
 
     cd ${OSH_PATH}
-    git reset --hard ${BARBICAN_STABLE_COMMIT}
+    # git reset --hard ${BARBICAN_STABLE_COMMIT}
     # Deploy required packages
     ./tools/deployment/common/install-packages.sh
     # Deploy Kubernetes

tools/run_pifpaf.sh

@@ -14,7 +14,7 @@ trap cleanup EXIT
 # variables for debugging purposes.
 set -ex
 if [ -z $(which pg_config) ]; then
-    sudo apt-get install libpq-dev -y
+    sudo apt-get install libpq-dev postgresql -y
 fi
 
 eval `pifpaf run postgresql`

tox.ini

@@ -1,11 +1,14 @@
 [tox]
-minversion = 2.3.1
+minversion = 3.4
+# Flag indicating to perform the packaging operation or not.
+# Set it to true when using tox for an application, instead of a library.
 skipsdist = True
-envlist = py{37,36},py{37,36}-{postgresql},functional,cover,pep8,bandit,docs
+envlist = py38,py38-{postgresql},functional,cover,pep8,bandit,docs
 
 [testenv]
+# Install the current package in development mode with develop mode
 usedevelop = True
-whitelist_externals = bash
+allowlist_externals = bash
                       find
                       rm
                       flake8
@@ -13,7 +16,21 @@ setenv = VIRTUAL_ENV={envdir}
          OS_TEST_PATH=./deckhand/tests/unit
          LANGUAGE=en_US
          LC_ALL=en_US.utf-8
-passenv = OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_TEST_TIMEOUT OS_TEST_LOCK_PATH OS_TEST_PATH http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY DECKHAND_IMAGE DECKHAND_TEST_URL DECKHAND_TEST_DIR
+passenv =
+    OS_STDOUT_CAPTURE
+    OS_STDERR_CAPTURE
+    OS_TEST_TIMEOUT
+    OS_TEST_LOCK_PATH
+    OS_TEST_PATH
+    http_proxy
+    HTTP_PROXY
+    https_proxy
+    HTTPS_PROXY
+    no_proxy
+    NO_PROXY
+    DECKHAND_IMAGE
+    DECKHAND_TEST_URL
+    DECKHAND_TEST_DIR
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
@@ -25,18 +42,20 @@ basepython=python3
 commands =
   {posargs}
 
-[testenv:py36]
+[testenv:py38]
-basepython = python3
 commands =
   {[testenv]commands}
   stestr run {posargs}
   stestr slowest
 
-[testenv:py36-postgresql]
+[testenv:py38-postgresql]
-basepython = python3
 commands =
     {[testenv]commands}
-    {toxinidir}/tools/run_pifpaf.sh '{posargs}'
+    bash {toxinidir}/tools/run_pifpaf.sh '{posargs}'
+allowlist_externals =
+  bash
+  find
+  rm
 
 [testenv:functional]
 basepython=python3
@@ -45,7 +64,7 @@ deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
   find . -type f -name "*.pyc" -delete
-  stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color '{posargs}'
+  stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color {posargs}
 
 [testenv:functional-dev]
 basepython=python3
@@ -106,7 +125,7 @@ commands =
 # [H904] Delay string interpolations at logging calls.
 enable-extensions = H106,H203,H204,H205,H210,H904
 # [E731] Do not assign a lambda expression, use a def. This reduces readability in some cases.
-ignore = E731,F405,H405,W504
+ignore = E731,F405,H405,W504,H306
 exclude =  .venv,.git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc,alembic/versions
 
 [testenv:docs]
@@ -114,7 +133,7 @@ basepython = python3
 deps =
   -r{toxinidir}/doc/requirements.txt
 commands =
-  {toxinidir}/tools/build-docs.sh
+  bash {toxinidir}/tools/build-docs.sh
 
 [testenv:releasenotes]
 basepython = python3
@@ -122,5 +141,5 @@ deps = -r{toxinidir}/doc/requirements.txt
 commands =
   rm -rf releasenotes/build
   sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
-whitelist_externals =
+allowlist_externals =
   rm