Airflow stable 2.6.2

This PS updates python modules and code to match Airflow 2.6.2

- bionic py36 gates  were removed
- python code corrected to match new modules versions
- selection of python modules versions was performed based on
  airflow-2.6.2 constraints
- a new pegleg validation gate was created based on airskiff deploy
  pipeline from treasuremap v1.9

Change-Id: Icc96deb0b745eaab7a19d83b00aad909dcd28c25

.zuul.yaml

@@ -12,10 +12,10 @@
 
 - project:
     templates:
-      - docs-on-readthedocs
+      # commented out because openstack-tox-docs fails on jammy zuul nodes
+      # - docs-on-readthedocs
       - openstack-python38-jobs
-      - openstack-python36-jobs
+      # - openstack-cover-jobs
-      - openstack-cover-jobs
       # - release-notes-jobs-python3
     vars:
       rtd_webhook_id: '38574'
@@ -34,17 +34,21 @@
     check:
       jobs:
         - openstack-tox-pep8-focal
-        # - pegleg-dependency-vulnerability-check
+        - openstack-tox-cover-focal
+        - openstack-tox-docs-focal
+        - pegleg-dependency-vulnerability-check
+        - airship-pegleg-validate-airskiff-manifests-focal
         - airship-pegleg-docker-build-gate-ubuntu_focal
-        - airship-pegleg-docker-build-gate-ubuntu_bionic
         - airship-pegleg-lint-yaml
 
     gate:
       jobs:
         - openstack-tox-pep8-focal
+        - openstack-tox-cover-focal
+        - openstack-tox-docs-focal
         - pegleg-dependency-vulnerability-check
+        - airship-pegleg-validate-airskiff-manifests-focal
         - airship-pegleg-docker-build-gate-ubuntu_focal
-        - airship-pegleg-docker-build-gate-ubuntu_bionic
         - airship-pegleg-lint-yaml
     post:
       jobs:
@@ -69,6 +73,17 @@
     parent: openstack-tox-pep8
     nodeset: openstack-single-node-focal
 
+- job:
+    name: openstack-tox-docs-focal
+    parent: openstack-tox-docs
+    nodeset: openstack-single-node-focal
+
+- job:
+    name: openstack-tox-cover-focal
+    parent: openstack-tox-cover
+    nodeset: openstack-single-node-focal
+
+
 - job:
     name: airship-pegleg-lint-yaml
     voting: true
@@ -79,19 +94,43 @@
       - ^.*\.rst$
       - ^doc/.*$
 
-# - job:
+- job:
-#     name: pegleg-dependency-vulnerability-check
+    name: pegleg-dependency-vulnerability-check
-#     parent: tox-py38
+    parent: tox-py38
-#     voting: false
+    voting: false
-#     timeout: 600
+    timeout: 600
-#     nodeset: ubuntu-focal
+    nodeset: ubuntu-focal
-#     vars:
+    vars:
-#       tox_envlist: safety
+      tox_envlist: safety
-#       bindep_profile: test py38
+      bindep_profile: test py38
+
+- job:
+    name: airship-pegleg-validate-airskiff-manifests-focal
+    nodeset: airship-pegleg-single-node-focal
+    description: |
+      Test pegleg Airskiff site manifest validation
+    timeout: 9600
+    voting: true
+    pre-run:
+      - tools/gate/playbooks/airskiff-reduce-site.yaml
+      - tools/gate/playbooks/git-config.yaml
+    run: tools/gate/playbooks/airskiff-validate.yaml
+    post-run: tools/gate/playbooks/debug-report.yaml
+    required-projects:
+      - airship/treasuremap
+    vars:
+      CLONE_PEGLEG: false
+      HTK_COMMIT: 97ce6d7d8e9a090c748800d69a57bbd9af698b60
+      PEGLEG_IMAGE_DISTRO: ubuntu_focal
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+
 
 - job:
     name: airship-pegleg-docker-build-gate-ubuntu_focal
-    timeout: 3600
+    timeout: 7200
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: airship-pegleg-single-node-focal
     irrelevant-files:
@@ -105,7 +144,7 @@
 
 - job:
     name: airship-pegleg-docker-publish-ubuntu_focal
-    timeout: 3600
+    timeout: 7200
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: airship-pegleg-single-node-focal
     secrets:
@@ -121,39 +160,7 @@
           commit: true
         static:
           - latest
-
+          - airflow_2.6.2
-- job:
-    name: airship-pegleg-docker-build-gate-ubuntu_bionic
-    timeout: 3600
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: airship-pegleg-single-node-focal
-    irrelevant-files:
-      - '^doc/.*'
-    vars:
-      publish: false
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          patch_set: true
-
-- job:
-    name: airship-pegleg-docker-publish-ubuntu_bionic
-    timeout: 3600
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: airship-pegleg-single-node-focal
-    secrets:
-      - airship_pegleg_quay_creds
-    irrelevant-files:
-      - '^doc/.*'
-    vars:
-      publish: true
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          branch: true
-          commit: true
-        static:
-          - latest
 
 - secret:
     name: airship_pegleg_quay_creds

requirements-direct.txt

@@ -1,16 +1,14 @@
-click==7.1.2
 jsonschema
-cryptography==3.4.8
+python-dateutil
-python-dateutil==2.8.1
 GitPython
 docker
-pylibyaml==0.1.0
+pylibyaml
-PyYAML<=5.4.1
+PyYAML
-deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5#egg=deckhand
+deckhand @ git+https://opendev.org/airship/deckhand.git@37ba567da5bf7f146f54a468478a025c70793574#egg=deckhand
-shipyard-client @ git+https://opendev.org/airship/shipyard.git@7ea83c25efc4e689e5cb0df19614d9ca538eda22#egg=shipyard-client&subdirectory=src/bin/shipyard_client
+shipyard-client @ git+https://opendev.org/airship/shipyard.git@81066ae98fe2afd3a9c8c5c8556e9438ac47d5a2#egg=shipyard-client&subdirectory=src/bin/shipyard_client
-promenade @ git+https://opendev.org/airship/promenade.git@16debd81747fcde8e0628e141ee67d697b06c46d#egg=promenade
+promenade @ git+https://opendev.org/airship/promenade.git@69a74590e76e810916f7780fc525c63ec58c7dc1#egg=promenade
-oslo.i18n
+oslo.i18n==6.0.0
-oslo.utils<=4.12.3
+oslo.utils==4.12.3
 six
 debtcollector
 zipp

requirements-frozen-bionic.txt

@@ -1,177 +0,0 @@
-alabaster==0.7.13
-alembic==1.4.3
-amqp==2.6.1
-argcomplete==3.0.8
-arrow==0.17.0
-attrs==22.2.0
-autopage==0.5.1
-Babel==2.11.0
-bcrypt==4.0.1
-Beaker==1.12.0
-cachetools==4.2.4
-certifi==2023.5.7
-cffi==1.15.1
-chardet==3.0.4
-charset-normalizer==3.0.1
-click==7.1.2
-click-default-group==1.2
-cliff==3.10.1
-cmd2==2.4.3
-configparser==5.2.0
-coverage==5.3
-cryptography==3.4.8
-debtcollector==2.5.0
-Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5
-decorator==4.4.2
-deepdiff==5.7.0
-distlib==0.3.6
-dnspython==2.2.1
-docker==5.0.3
-docutils==0.17
-dogpile.cache==1.2.1
-dulwich==0.20.50
-eventlet==0.33.3
-extras==1.0.0
-falcon==3.1.1
-fasteners==0.18
-filelock==3.4.1
-fixtures==3.0.0
-flake8==3.8.4
-Flask==1.1.0
-futurist==2.4.1
-gitdb==4.0.9
-GitPython==3.1.18
-google-auth==2.19.1
-greenlet==2.0.2
-hacking==4.1.0
-html5lib==0.9999999
-httpexceptor==1.4.0
-idna==2.10
-imagesize==1.4.1
-importlib-metadata==2.1.3
-importlib-resources==5.4.0
-iso8601==1.1.0
-itsdangerous==2.0.1
-Jinja2==2.10.3
-jsonpath-ng==1.5.3
-jsonpath-rw==1.4.0
-jsonpath-rw-ext==1.2.2
-jsonpickle==1.4.1
-jsonschema==3.2.0
-keystoneauth1==5.1.1
-keystonemiddleware==9.5.0
-kombu==4.6.11
-kubernetes==26.1.0
-Mako==1.1.6
-MarkupSafe==2.0.1
-mccabe==0.6.1
-mock==5.0.2
-msgpack==1.0.5
-netaddr==0.8.0
-netifaces==0.11.0
-networkx==2.5.1
-nose==1.3.7
-oauthlib==3.2.2
-ordered-set==4.0.2
-os-service-types==1.7.0
-oslo.cache==2.10.1
-oslo.concurrency==4.5.1
-oslo.config==8.7.1
-oslo.context==4.1.0
-oslo.db==10.0.0
-oslo.i18n==5.1.0
-oslo.log==4.6.0
-oslo.messaging==12.13.0
-oslo.metrics==0.4.0
-oslo.middleware==4.4.0
-oslo.policy==3.10.1
-oslo.serialization==4.2.0
-oslo.service==2.8.0
-oslo.utils==4.12.3
-packaging==21.3
-Paste==3.5.0
-PasteDeploy==2.1.1
-PasteScript==3.3.0
-pbr==5.5.1
-pip==21.3.1
-platformdirs==2.4.0
-ply==3.11
-prettytable==2.5.0
-promenade @ git+https://opendev.org/airship/promenade.git@16debd81747fcde8e0628e141ee67d697b06c46d
-prometheus-client==0.17.0
-psycopg2-binary==2.9.6
-PTable==0.9.2
-pyasn1==0.5.0
-pyasn1-modules==0.3.0
-pycadf==3.1.1
-pycodestyle==2.6.0
-pycparser==2.21
-pyflakes==2.2.0
-Pygments==2.14.0
-pylibyaml==0.1.0
-pymongo==4.1.1
-pyparsing==2.4.7
-pyperclip==1.8.2
-pyproject-api==0.1.0
-pyrsistent==0.18.0
-python-barbicanclient==5.2.0
-python-dateutil==2.8.1
-python-editor==1.0.4
-python-keystoneclient==4.5.0
-python-memcached==1.59
-python-mimeparse==1.6.0
-python-subunit==1.4.0
-pytz==2023.3
-PyYAML==5.4.1
-reno==4.0.0
-repoze.lru==0.7
-requests==2.23.0
-requests-oauthlib==1.3.1
-resolver==0.2.1
-rfc3986==1.5.0
-Routes==2.5.1
-rsa==4.9
-selector==0.10.1
-setuptools==45.2.0
-shipyard-client @ git+https://opendev.org/airship/shipyard.git@7ea83c25efc4e689e5cb0df19614d9ca538eda22#subdirectory=src/bin/shipyard_client
-simplejson==3.19.1
-six==1.16.0
-smmap==5.0.0
-snowballstemmer==2.2.0
-Sphinx==3.3.1
-sphinx-rtd-theme==0.5.0
-sphinxcontrib-applehelp==1.0.2
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.0
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==1.3.20
-sqlalchemy-migrate==0.13.0
-sqlparse==0.4.4
-statsd==4.0.1
-stevedore==3.5.2
-Tempita==0.5.2
-testrepository==0.0.20
-testresources==2.0.1
-testscenarios==0.5.0
-testtools==2.5.0
-tiddlyweb==2.4.3
-tomli==1.2.3
-tomlkit==0.11.6
-typing-extensions==3.7.4.3
-urllib3==1.25.11
-uWSGI==2.0.21
-vine==1.3.0
-virtualenv==20.16.2
-wcwidth==0.2.6
-WebOb==1.8.7
-websocket-client==1.3.1
-Werkzeug==0.16.1
-wheel==0.37.1
-wrapt==1.15.0
-wsgi-intercept==1.11.0
-xmltodict==0.13.0
-yappi==1.4.0
-yq==3.2.1
-zipp==3.6.0

requirements-frozen.txt

@@ -1,176 +1,144 @@
-alabaster==0.7.13
+alembic==1.11.1
-alembic==1.4.3
+amqp==5.1.1
-amqp==2.6.1
+arrow==1.2.3
-argcomplete==3.0.8
-arrow==0.17.0
 attrs==23.1.0
 autopage==0.5.1
-Babel==2.12.1
+backports.zoneinfo==0.2.1
+barbican==14.0.2
 bcrypt==4.0.1
-Beaker==1.12.0
+Beaker==1.12.1
 cachetools==5.3.1
+castellan==4.1.0
 certifi==2023.5.7
 cffi==1.15.1
-chardet==3.0.4
 charset-normalizer==3.1.0
-click==7.1.2
+click==8.1.3
-click-default-group==1.2
+click-default-group==1.2.2
-cliff==3.10.1
+cliff==4.3.0
 cmd2==2.4.3
-configparser==5.3.0
+cryptography==40.0.2
-coverage==5.3
-cryptography==3.4.8
 debtcollector==2.5.0
-Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5
+Deckhand @ git+https://opendev.org/airship/deckhand.git@37ba567da5bf7f146f54a468478a025c70793574
 decorator==5.1.1
-deepdiff==5.8.1
+deepdiff==6.3.1
-distlib==0.3.6
 dnspython==2.3.0
-docker==5.0.3
+docker==6.1.3
-docutils==0.17
+dogpile.cache==1.2.2
-dogpile.cache==1.2.1
-dulwich==0.21.5
 eventlet==0.33.3
-extras==1.0.0
 falcon==3.1.1
 fasteners==0.18
-filelock==3.12.0
+fixtures==4.1.0
-fixtures==3.0.0
+Flask==2.2.5
-flake8==3.8.4
-Flask==1.1.0
 futurist==2.4.1
 gitdb==4.0.10
 GitPython==3.1.31
-google-auth==2.19.1
+google-auth==2.20.0
 greenlet==2.0.2
-hacking==4.1.0
 html5lib==0.9999999
 httpexceptor==1.4.0
-idna==2.10
+idna==3.4
-imagesize==1.4.1
+importlib-metadata==4.13.0
-importlib-metadata==2.1.3
+importlib-resources==5.12.0
 iso8601==2.0.0
-itsdangerous==2.0.1
+itsdangerous==2.1.2
-Jinja2==2.10.3
+Jinja2==3.1.2
 jsonpath-ng==1.5.3
-jsonpath-rw==1.4.0
+jsonpickle==3.0.1
-jsonpath-rw-ext==1.2.2
+jsonschema==4.17.3
-jsonpickle==1.4.1
+keystoneauth1==4.3.1
-jsonschema==3.2.0
+keystonemiddleware==10.4.0
-keystoneauth1==5.1.1
+kombu==5.3.0
-keystonemiddleware==10.3.0
+kubernetes==23.6.0
-kombu==4.6.11
+ldap3==2.9.1
-kubernetes==26.1.0
+logutils==0.3.5
 Mako==1.2.4
-MarkupSafe==2.0.1
+MarkupSafe==2.1.3
-mccabe==0.6.1
-mock==5.0.2
 msgpack==1.0.5
 netaddr==0.8.0
 netifaces==0.11.0
 networkx==3.1
-nose==1.3.7
 oauthlib==3.2.2
 ordered-set==4.1.0
 os-service-types==1.7.0
-oslo.cache==2.10.1
+oslo.cache==3.4.0
 oslo.concurrency==5.1.1
-oslo.config==8.7.1
+oslo.config==8.6.0
-oslo.context==4.1.0
+oslo.context==4.0.0
-oslo.db==10.0.0
+oslo.db==11.2.0
 oslo.i18n==6.0.0
-oslo.log==4.6.0
+oslo.log==4.8.0
-oslo.messaging==12.13.0
+oslo.messaging==14.3.1
 oslo.metrics==0.6.0
-oslo.middleware==4.4.0
+oslo.middleware==5.1.1
 oslo.policy==3.10.1
-oslo.serialization==4.2.0
+oslo.serialization==4.1.0
 oslo.service==3.1.1
+oslo.upgradecheck==2.1.1
 oslo.utils==4.12.3
+oslo.versionedobjects==3.1.0
 packaging==21.3
-Paste==3.5.0
+Paste==3.5.3
 PasteDeploy==3.0.1
 PasteScript==3.3.0
-pbr==5.5.1
+pbr==5.11.1
+pecan==1.5.1
 pip==23.1.2
-platformdirs==3.5.1
+pkgutil_resolve_name==1.3.10
 ply==3.11
-prettytable==3.7.0
+prettytable==3.8.0
-promenade @ git+https://opendev.org/airship/promenade.git@16debd81747fcde8e0628e141ee67d697b06c46d
+promenade @ git+https://opendev.org/airship/promenade.git@69a74590e76e810916f7780fc525c63ec58c7dc1
 prometheus-client==0.17.0
 psycopg2-binary==2.9.6
 PTable==0.9.2
-pyasn1==0.5.0
+pyasn1==0.4.8
-pyasn1-modules==0.3.0
+pyasn1-modules==0.2.8
 pycadf==3.1.1
-pycodestyle==2.6.0
 pycparser==2.21
-pyflakes==2.2.0
-Pygments==2.14.0
 pylibyaml==0.1.0
-pymongo==4.3.3
+pyOpenSSL==23.2.0
-pyparsing==2.4.7
+pyparsing==3.0.9
 pyperclip==1.8.2
-pyproject_api==1.5.0
 pyrsistent==0.19.3
-python-barbicanclient==5.2.0
+python-barbicanclient==5.4.0
-python-dateutil==2.8.1
+python-dateutil==2.8.2
-python-editor==1.0.4
 python-keystoneclient==5.1.0
 python-memcached==1.59
 python-mimeparse==1.6.0
-python-subunit==1.4.0
 pytz==2023.3
-PyYAML==5.4.1
+PyYAML==6.0
-reno==4.0.0
 repoze.lru==0.7
-requests==2.23.0
+requests==2.31.0
 requests-oauthlib==1.3.1
 resolver==0.2.1
-rfc3986==2.0.0
+rfc3986==1.5.0
 Routes==2.5.1
 rsa==4.9
 selector==0.10.1
-setuptools==45.2.0
+setuptools==67.7.2
-shipyard-client @ git+https://opendev.org/airship/shipyard.git@7ea83c25efc4e689e5cb0df19614d9ca538eda22#subdirectory=src/bin/shipyard_client
+shipyard-client @ git+https://opendev.org/airship/shipyard.git@81066ae98fe2afd3a9c8c5c8556e9438ac47d5a2#subdirectory=src/bin/shipyard_client
 simplejson==3.19.1
 six==1.16.0
 smmap==5.0.0
-snowballstemmer==2.2.0
+SQLAlchemy==1.4.48
-Sphinx==3.3.1
-sphinx-rtd-theme==0.5.0
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==1.3.20
 sqlalchemy-migrate==0.13.0
 sqlparse==0.4.4
 statsd==4.0.1
 stevedore==5.1.0
 Tempita==0.5.2
-testrepository==0.0.20
 testresources==2.0.1
 testscenarios==0.5.0
-testtools==2.5.0
+testtools==2.6.0
 tiddlyweb==2.4.3
-tomli==2.0.1
+typing_extensions==4.6.3
-tomlkit==0.11.8
+tzdata==2023.3
-typing-extensions==3.7.4.3
+urllib3==1.26.16
-urllib3==1.25.11
+uWSGI==2.0.22
-uWSGI==2.0.21
+vine==5.0.0
-vine==1.3.0
-virtualenv==20.23.0
 wcwidth==0.2.6
 WebOb==1.8.7
 websocket-client==1.5.3
-Werkzeug==0.16.1
+Werkzeug==2.2.3
 wheel==0.40.0
 wrapt==1.15.0
-wsgi-intercept==1.11.0
-xmltodict==0.13.0
 yappi==1.4.0
-yq==3.2.1
 zipp==3.15.0

tools/gate/playbooks/airskiff-reduce-site.yaml

@@ -0,0 +1,23 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Overwrite Armada manifest
+      shell: |
+        git checkout v1.9
+        mv tools/gate/manifests/full-site.yaml \
+            type/skiff/manifests/full-site.yaml
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"

tools/gate/playbooks/airskiff-validate.yaml

@@ -0,0 +1,152 @@
+# Copyright 2019, AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  roles:
+    - clear-firewall
+    - bindep
+    - disable-systemd-resolved
+    - ensure-python
+    - ensure-pip
+    - ensure-docker
+
+  tasks:
+
+    - name: Install Packaging python module for tools/airship
+      block:
+        - pip:
+            name: packaging
+            version: 23.1
+            executable: pip3
+      become: True
+
+    - name: Clone Required Repositories
+      shell: |
+        export CLONE_PEGLEG={{ CLONE_PEGLEG }}
+        export OSH_INFRA_COMMIT={{ HTK_COMMIT }}
+        ./tools/deployment/airskiff/developer/000-clone-dependencies.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+
+    - name: Setup Apparmor
+      shell: |
+        ./tools/deployment/airskiff/developer/009-setup-apparmor.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+
+    - name: Deploy Kubernetes with Minikube
+      shell: |
+        ./tools/deployment/airskiff/developer/010-deploy-k8s.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+
+
+    - name: Build all charts locally
+      shell: |
+        set -ex
+        ./tools/deployment/airskiff/developer/015-make-all-charts.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
+    - name: Build Pegleg with submitted changes
+      shell: |
+        set -ex
+        export DISTRO={{ PEGLEG_IMAGE_DISTRO }}
+        make images
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      become: yes
+
+
+    - name: Use locally built images in manifests
+      shell: |
+        set -ex
+        docker rm registry --force || true
+        docker run -d -p 5000:5000 --restart=always --name registry registry:2
+        docker tag quay.io/airshipit/pegleg:latest-ubuntu_focal localhost:5000/pegleg:latest-ubuntu_focal
+        docker push localhost:5000/pegleg:latest-ubuntu_focal
+        sed -i "s#quay.io/airshipit/pegleg:latest-ubuntu_focal#localhost:5000/pegleg:latest-ubuntu_focal#g" ./global/software/config/versions.yaml
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
+    - name: Start artifactory
+      shell: |
+        set -ex
+        # start http server with artifacts
+        docker rm artifacts --force || true
+        docker run --name artifacts -p 8282:80 -v $(pwd)/../artifacts:/usr/share/nginx/html -d nginx
+        sleep 10
+        curl --verbose -I http://control-plane.minikube.internal:8282/memcached.tgz
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
+    - name: List sites
+      shell: |
+        set -xe
+        ./tools/airship pegleg --verbose site    -r . list
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
+    - name: Lint Airskiff site
+      shell: |
+        set -xe
+        ./tools/airship pegleg --verbose site    -r . lint airskiff -w P001 -w P003
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
+    - name: Validate Airskiff site
+      shell: |
+        set -xe
+        ./tools/airship pegleg --verbose site    -r . render  airskiff --validate
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
+    - name: Render Airskiff site
+      shell: |
+        set -xe
+        ./tools/airship pegleg  --verbose site -r . render airskiff -o airskiff.yaml
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
+    - name: Show Airskiff site
+      shell: |
+        set -xe
+        ./tools/airship pegleg  --verbose site -r . show airskiff
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
+    - name: Collect Airskiff site
+      shell: |
+        set -xe
+        ./tools/airship pegleg  --verbose site -r . collect airskiff -s peggles
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
+    - name: Stop artifactory
+      shell: |
+        set -ex
+        # terminate artifactory
+        docker rm artifacts --force || true
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+      become: yes

tools/gate/playbooks/debug-report.yaml

@@ -0,0 +1,34 @@
+# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Build debug report
+      shell: |
+        ./tools/gate/debug-report.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+
+    - name: Extract debug report
+      shell: |
+        mkdir -p /tmp/debug
+        tar -xf debug-* -C /tmp/debug
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+
+    - name: Pull logs to executor
+      synchronize:
+        src: /tmp/debug
+        dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+        mode: pull

tools/gate/playbooks/git-config.yaml

@@ -0,0 +1,25 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Git config
+      shell: |
+        set -xe;
+        tee .git/config << EOF
+        [remote "origin"]
+        url = https://opendev.org/airship/treasuremap.git
+        EOF
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"

tools/gate/playbooks/roles

@@ -0,0 +1 @@
+../roles

tools/gate/roles/deploy-package/defaults/main.yml

@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+proxy:
+  http: null
+  https: null
+  noproxy: null
+...

tools/gate/roles/deploy-package/tasks/dist.yaml

@@ -0,0 +1,46 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: managing distro packages for ubuntu
+  become: true
+  become_user: root
+  when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
+  vars:
+    state: present
+  apt:
+    name: "{{ item }}"
+    state: "{{ state }}"
+  with_items: "{{ packages.deb }}"
+
+- name: managing distro packages for centos
+  become: true
+  become_user: root
+  when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux'
+  vars:
+    state: present
+  yum:
+    name: "{{ item }}"
+    state: "{{ state }}"
+  with_items: "{{ packages.rpm }}"
+
+- name: managing distro packages for fedora
+  become: true
+  become_user: root
+  when: ansible_distribution == 'Fedora'
+  vars:
+    state: present
+  dnf:
+    name: "{{ item }}"
+    state: "{{ state }}"
+  with_items: "{{ packages.rpm }}"
+...

tools/gate/roles/deploy-package/tasks/pip.yaml

@@ -0,0 +1,27 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: managing pip packages
+  become: true
+  become_user: root
+  environment:
+    http_proxy: "{{ proxy.http }}"
+    https_proxy: "{{ proxy.https }}"
+    no_proxy: "{{ proxy.noproxy }}"
+  vars:
+    state: present
+  pip:
+    name: "{{ item }}"
+    state: "{{ state }}"
+  with_items: "{{ packages }}"
+...

tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml

@@ -0,0 +1,37 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Disable systemd-resolved service
+  systemd:
+    state: stopped
+    enabled: no
+    masked: yes
+    daemon_reload: yes
+    name: systemd-resolved
+  become: yes
+
+- name: Remove local stub dns from resolv.conf, if it exists
+  lineinfile:
+    path: /etc/resolv.conf
+    state: absent
+    regexp: '^nameserver.*127.0.0.1'
+  become: yes
+
+- name: Add upstream nameservers in resolv.conf
+  blockinfile:
+    path: /etc/resolv.conf
+    block: |
+      nameserver 8.8.8.8
+      nameserver 8.8.4.4
+  become: yes

tools/gate/roles/disable-systemd-resolved/tasks/main.yaml

@@ -0,0 +1,15 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- include: disable-systemd-resolved.yaml

tox.ini

@@ -30,16 +30,6 @@ commands =
   bash -c "{toxinidir}/tools/install-cfssl.sh"
   bash -c "{toxinidir}/tools/gate/run-unit-tests.sh '{posargs}'"
 
-[testenv:py36]
-deps =
-  -r{toxinidir}/requirements-frozen-bionic.txt
-  -r{toxinidir}/test-requirements.txt
-allowlist_externals =
-  bash
-  find
-commands =
-  {[testenv]commands}
-
 
 [testenv:py38]
 allowlist_externals =
@@ -103,17 +93,6 @@ allowlist_externals =
   bash
   find
 
-[testenv:freeze-bionic]
-recreate = True
-allowlist_externals=
-  rm
-  sh
-deps=
-  -r{toxinidir}/requirements-direct.txt
-commands=
-  rm -f {toxinidir}/requirements-frozen-bionic.txt
-  sh -c "pip freeze --all | grep -vE 'pegleg|pyinotify|pkg-resources' > requirements-frozen-bionic.txt"
-
 [testenv:freeze]
 recreate = True
 allowlist_externals=
@@ -121,6 +100,7 @@ allowlist_externals=
   sh
 deps=
   -r{toxinidir}/requirements-direct.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
 commands=
   rm -f {toxinidir}/requirements-frozen.txt
   sh -c "pip freeze --all | grep -vE 'pegleg|pyinotify|pkg-resources' > requirements-frozen.txt"
@@ -130,7 +110,6 @@ commands=
 [testenv:releasenotes]
 basepython = python3
 deps=
-    -r{toxinidir}/requirements-frozen.txt
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/doc/requirements.txt
 commands =

vcs-requirements.env

@@ -1,3 +1,3 @@
-DECKHAND_VERSION=commit/06b1631d8a31a1e7147e8751025d0ef2ff4b31f5
+DECKHAND_VERSION=commit/37ba567da5bf7f146f54a468478a025c70793574
-SHIPYARD_VERSION=commit/7ea83c25efc4e689e5cb0df19614d9ca538eda22
+SHIPYARD_VERSION=commit/81066ae98fe2afd3a9c8c5c8556e9438ac47d5a2
-PROMENADE_VERSION=commit/16debd81747fcde8e0628e141ee67d697b06c46d
+PROMENADE_VERSION=commit/69a74590e76e810916f7780fc525c63ec58c7dc1