Airflow stable 2.6.2

This PS updates python modules and code to match Airflow 2.6.2:

- bionic py36 gates  were removed
- python code corrected to match new modules versions
- selection of python modules versions was perfrmed based on
  airflow-2.6.2 constraints
- postgresql image updated to 14.8

Change-Id: Ibdcc75e600166c20b842508aa5539587cca466f0

.zuul.yaml

@@ -14,7 +14,6 @@
     templates:
       - docs-on-readthedocs
       - openstack-python38-jobs
-      - openstack-python36-jobs
     vars:
       rtd_webhook_id: '38809'
       rtd_project_name: 'airship-drydock'
@@ -26,7 +25,6 @@
         - airship-drydock-chart-build-gate
         - airship-drydock-chart-build-latest-htk
         - airship-drydock-docker-build-gate-ubuntu_focal
-        - airship-drydock-docker-build-gate-ubuntu_bionic
 
     gate:
       jobs:
@@ -35,11 +33,10 @@
         - airship-drydock-baclient-test
         - airship-drydock-chart-build-gate
         - airship-drydock-docker-build-gate-ubuntu_focal
-        - airship-drydock-docker-build-gate-ubuntu_bionic
+
     post:
       jobs:
         - airship-drydock-docker-publish-ubuntu_focal
-        - airship-drydock-docker-publish-ubuntu_bionic
         - drydock-upload-git-mirror
 
 - nodeset:
@@ -84,7 +81,7 @@
     run: tools/gate/playbooks/build-charts.yaml
     nodeset: airship-drydock-single-node-focal
     vars:
-      HTK_COMMIT: a7cd689280cdbc0acd04a7a1b745941260e8700b
+      HTK_COMMIT: 97ce6d7d8e9a090c748800d69a57bbd9af698b60
 
 - job:
     name: airship-drydock-chart-build-latest-htk
@@ -113,21 +110,6 @@
           patch_set: true
 
 
-- job:
-    name: airship-drydock-docker-build-gate-ubuntu_bionic
-    timeout: 3600
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: airship-drydock-single-node
-    irrelevant-files:
-      - '^doc/.*'
-      - '^charts/.*'
-    vars:
-      publish: false
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          patch_set: true
-
 
 - job:
     name: airship-drydock-docker-publish-ubuntu_focal
@@ -148,26 +130,7 @@
           commit: true
         static:
           - latest
-
+          - airflow_2.6.2
-- job:
-    name: airship-drydock-docker-publish-ubuntu_bionic
-    timeout: 3600
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: airship-drydock-single-node-focal
-    secrets:
-      - airship_drydock_quay_creds
-    irrelevant-files:
-      - '^doc/.*'
-      - '^charts/.*'
-    vars:
-      publish: true
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          branch: true
-          commit: true
-        static:
-          - latest
 
 
 - secret:

charts/drydock/values.yaml

@@ -35,7 +35,7 @@ images:
     ks_user: docker.io/openstackhelm/heat:newton
     ks_service: docker.io/openstackhelm/heat:newton
     ks_endpoints: docker.io/openstackhelm/heat:newton
-    drydock_db_init: docker.io/postgres:14.6
+    drydock_db_init: docker.io/postgres:14.8
     drydock_db_cleanup: quay.io/airshipit/drydock:master
     drydock_db_sync: quay.io/airshipit/drydock:master
   pull_policy: "IfNotPresent"

doc/source/_static/drydock.conf.sample

@@ -78,7 +78,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>
 
-# ID of the trust to use as a trustee use (string value)
+# Trust ID (string value)
 #trust_id = <None>
 
 # Optional domain ID to use with v3 and v2 parameters. It will be used for both

etc/drydock/drydock.conf.sample

@@ -78,7 +78,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>
 
-# ID of the trust to use as a trustee use (string value)
+# Trust ID (string value)
 #trust_id = <None>
 
 # Optional domain ID to use with v3 and v2 parameters. It will be used for both

python/drydock_provisioner/drivers/node/maasdriver/actions/node.py

@@ -296,7 +296,7 @@ class DestroyNode(BaseMaasAction):
                                              ctx_type='node')
                     self.task.success(focus=n.get_id())
                     continue
-                elif type(machine) == maas_rack.RackController:
+                elif type(machine) is maas_rack.RackController:
                     msg = "Cannot delete rack controller {}.".format(n.name)
                     self.logger.info(msg)
                     self.task.add_status_msg(msg=msg,
@@ -1130,7 +1130,7 @@ class IdentifyNode(BaseMaasAction):
                                              error=True,
                                              ctx=n.name,
                                              ctx_type='node')
-                elif type(machine) == maas_machine.Machine:
+                elif type(machine) is maas_machine.Machine:
                     machine.update_identity(
                         n,
                         domain=n.get_domain(site_design),
@@ -1144,7 +1144,7 @@ class IdentifyNode(BaseMaasAction):
                                              ctx=n.name,
                                              ctx_type='node')
                     self.task.success(focus=n.get_id())
-                elif type(machine) == maas_rack.RackController:
+                elif type(machine) is maas_rack.RackController:
                     msg = "Rack controller %s identified in MaaS" % n.name
                     self.logger.debug(msg)
                     self.task.add_status_msg(msg=msg,
@@ -1219,7 +1219,7 @@ class ConfigureHardware(BaseMaasAction):
                 self.logger.debug("Locating node %s for commissioning" %
                                   (n.name))
                 machine = find_node_in_maas(self.maas_client, n)
-                if type(machine) == maas_rack.RackController:
+                if type(machine) is maas_rack.RackController:
                     msg = "Located node %s in MaaS as rack controller. Skipping." % (
                         n.name)
                     self.logger.info(msg)

python/requirements-direct.txt

@@ -7,13 +7,19 @@ falcon
 iso8601
 jinja2
 jsonschema
-keystoneauth1
+keystoneauth1==4.3.1
 keystonemiddleware
 libvirt-python
 netaddr
+mock
 oauthlib
-oslo.config
+oslo.config==8.6.0
-oslo.policy
+oslo.context==4.0.0
+oslo.log==4.8.0
+oslo.policy==3.10.1
+oslo.serialization==4.1.0
+oslo.utils==4.12.3
+oslo.db==11.2.0
 oslo.versionedobjects
 Paste
 PasteDeploy
@@ -22,9 +28,11 @@ PTable
 pyghmi
 pylibyaml
 pymongo
+python-memcached
 PyYAML
 redfish
 requests
-SQLAlchemy<=1.3.20
+responses
+SQLAlchemy
 ulid2
 uWSGI

python/requirements-frozen-bionic.txt

@@ -1,99 +0,0 @@
-alembic==1.7.7
-amqp==5.1.1
-attrs==22.2.0
-bcrypt==4.0.1
-Beaker==1.12.1
-cached-property==1.5.2
-cachetools==4.2.4
-certifi==2023.5.7
-cffi==1.15.1
-charset-normalizer==2.0.12
-click==8.0.4
-cryptography==40.0.2
-debtcollector==2.5.0
-decorator==5.1.1
-defusedxml==0.7.1
-dnspython==2.2.1
-dogpile.cache==1.2.1
-eventlet==0.33.3
-falcon==3.1.1
-fasteners==0.18
-fixtures==4.0.1
-futurist==2.4.1
-greenlet==2.0.2
-idna==3.4
-importlib-metadata==4.8.3
-importlib-resources==5.4.0
-iso8601==1.1.0
-Jinja2==3.0.3
-jsonpatch==1.32
-jsonpath-rw==1.4.0
-jsonpointer==2.3
-jsonschema==3.2.0
-keystoneauth1==5.1.2
-keystonemiddleware==9.5.0
-kombu==5.1.0
-libvirt-python==9.3.0
-Mako==1.1.6
-MarkupSafe==2.0.1
-msgpack==1.0.5
-netaddr==0.8.0
-netifaces==0.11.0
-oauthlib==3.2.2
-os-service-types==1.7.0
-oslo.cache==2.11.0
-oslo.concurrency==4.5.1
-oslo.config==8.8.0
-oslo.context==4.1.0
-oslo.i18n==5.1.0
-oslo.log==4.8.0
-oslo.messaging==12.14.0
-oslo.metrics==0.4.0
-oslo.middleware==4.5.1
-oslo.policy==3.12.1
-oslo.serialization==4.3.0
-oslo.service==2.8.0
-oslo.utils==4.13.0
-oslo.versionedobjects==2.6.0
-packaging==21.3
-Paste==3.5.3
-PasteDeploy==2.1.1
-pbr==5.11.1
-pip==21.3.1
-ply==3.11
-prometheus-client==0.16.0
-psycopg2-binary==2.9.6
-PTable==0.9.2
-pycadf==3.1.1
-pycparser==2.21
-pyghmi==1.5.61
-pylibyaml==0.1.0
-pymongo==4.1.1
-pyparsing==3.0.9
-pyrsistent==0.18.0
-python-dateutil==2.8.2
-python-keystoneclient==4.5.0
-pytz==2023.3
-PyYAML==6.0
-redfish==3.1.9
-repoze.lru==0.7
-requests==2.27.1
-requests-toolbelt==1.0.0
-requests-unixsocket==0.3.0
-rfc3986==1.5.0
-Routes==2.5.1
-setuptools==59.6.0
-six==1.16.0
-SQLAlchemy==1.3.20
-statsd==4.0.1
-stevedore==3.5.2
-typing_extensions==4.1.1
-ulid2==0.3.0
-urllib3==1.26.16
-uWSGI==2.0.21
-vine==5.0.0
-WebOb==1.8.7
-wheel==0.37.1
-wrapt==1.15.0
-yappi==1.4.0
-zipp==3.6.0

python/requirements-frozen.txt

@@ -1,131 +1,112 @@
-alabaster==0.7.13
+alembic==1.11.1
-alembic==1.10.4
 amqp==5.1.1
 attrs==23.1.0
-Babel==2.12.1
+backports.zoneinfo==0.2.1
-bandit==1.7.5
 bcrypt==4.0.1
 Beaker==1.12.1
-cachetools==5.3.0
+cachetools==5.3.1
-certifi==2022.12.7
+certifi==2023.5.7
 cffi==1.15.1
 charset-normalizer==3.1.0
-click==6.7
+click==8.1.3
-coverage==7.2.3
 cryptography==40.0.2
 debtcollector==2.5.0
 decorator==5.1.1
 defusedxml==0.7.1
 dnspython==2.3.0
-docutils==0.19
+dogpile.cache==1.2.2
-dogpile.cache==1.1.8
 eventlet==0.33.3
-exceptiongroup==1.1.1
 falcon==3.1.1
 fasteners==0.18
-fixtures==4.0.1
+fixtures==4.1.0
-flake8==6.0.0
 futurist==2.4.1
-gitdb==4.0.10
-GitPython==3.1.31
 greenlet==2.0.2
 idna==3.4
-imagesize==1.4.1
+importlib-metadata==4.13.0
-iniconfig==2.0.0
+importlib-resources==5.12.0
-iso8601==1.1.0
+iso8601==2.0.0
 Jinja2==3.1.2
 jsonpatch==1.32
 jsonpath-rw==1.4.0
 jsonpointer==2.3
-jsonschema==3.2.0
+jsonschema==4.17.3
-keystoneauth1==5.1.2
+keystoneauth1==4.3.1
-keystonemiddleware==10.2.0
+keystonemiddleware==10.4.0
-kombu==5.2.4
+kombu==5.3.0
-libvirt-python==9.2.0
+libvirt-python==9.6.0
 Mako==1.2.4
-markdown-it-py==2.2.0
+MarkupSafe==2.1.3
-MarkupSafe==2.1.2
+mock==5.1.0
-mccabe==0.7.0
-mdurl==0.1.2
-mock==5.0.1
 msgpack==1.0.5
 netaddr==0.8.0
 netifaces==0.11.0
 oauthlib==3.2.2
 os-service-types==1.7.0
-oslo.cache==3.3.1
+oslo.cache==3.4.0
 oslo.concurrency==5.1.1
-oslo.config==9.1.1
+oslo.config==8.6.0
-oslo.context==5.1.1
+oslo.context==4.0.0
+oslo.db==11.2.0
 oslo.i18n==6.0.0
-oslo.log==5.2.0
+oslo.log==4.8.0
-oslo.messaging==14.2.0
+oslo.messaging==14.3.1
 oslo.metrics==0.6.0
 oslo.middleware==5.1.1
-oslo.policy==4.1.1
+oslo.policy==3.10.1
-oslo.serialization==5.1.1
+oslo.serialization==4.1.0
 oslo.service==3.1.1
-oslo.utils==6.1.0
+oslo.utils==4.12.3
 oslo.versionedobjects==3.1.0
-packaging==23.1
+packaging==21.3
-Paste==3.5.2
+Paste==3.5.3
 PasteDeploy==3.0.1
 pbr==5.11.1
-pip==23.0.1
+pip==23.1.2
-pluggy==1.0.0
+pkgutil_resolve_name==1.3.10
 ply==3.11
-prometheus-client==0.16.0
+prometheus-client==0.17.0
 psycopg2-binary==2.9.6
 PTable==0.9.2
 pycadf==3.1.1
-pycodestyle==2.10.0
 pycparser==2.21
-pyflakes==3.0.1
+pyghmi==1.5.62
-pyghmi==1.5.61
-Pygments==2.15.1
 pylibyaml==0.1.0
-pymongo==3.10.1
+pymongo==4.3.3
 pyparsing==3.0.9
 pyrsistent==0.19.3
-pytest==7.3.1
-pytest-cov==4.0.0
-pytest-mock==3.10.0
 python-dateutil==2.8.2
 python-keystoneclient==5.1.0
+python-memcached==1.59
 pytz==2023.3
 PyYAML==6.0
-redfish==3.1.9
+redfish==3.2.1
 repoze.lru==0.7
-requests==2.29.0
+requests==2.31.0
-requests-toolbelt==0.10.1
+requests-toolbelt==1.0.0
 requests-unixsocket==0.3.0
 responses==0.23.1
-rfc3986==2.0.0
+rfc3986==1.5.0
-rich==13.3.4
 Routes==2.5.1
-setuptools==56.0.0
+setuptools==67.7.2
 six==1.16.0
-smmap==5.0.0
+SQLAlchemy==1.4.48
-snowballstemmer==2.2.0
+sqlalchemy-migrate==0.13.0
-Sphinx==6.2.1
+sqlparse==0.4.4
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==1.3.21
 statsd==4.0.1
-stevedore==5.0.0
+stevedore==5.1.0
-tomli==2.0.1
+Tempita==0.5.2
-types-PyYAML==6.0.12.9
+testresources==2.0.1
-typing_extensions==4.5.0
+testscenarios==0.5.0
-ulid2==0.1.1
+testtools==2.6.0
-urllib3==1.26.15
+types-PyYAML==6.0.12.10
-uWSGI==2.0.21
+typing_extensions==4.6.3
+tzdata==2023.3
+ulid2==0.3.0
+urllib3==1.26.16
+uWSGI==2.0.22
 vine==5.0.0
 WebOb==1.8.7
 wheel==0.40.0
 wrapt==1.15.0
-yapf==0.33.0
 yappi==1.4.0
+zipp==3.15.0

python/test-requirements.txt

@@ -1,15 +1,7 @@
-# tests
-click
-falcon==3.1.1
-jsonschema==3.2.0
-mock==5.0.1
-pylibyaml==0.1.0
-pymongo
 pytest >= 3.0
 pytest-cov==4.0.0
 pytest-mock
-responses
+
-ulid2
 
 # tools
 yapf

python/tests/postgres/start_postgres.sh

@@ -22,11 +22,11 @@ then
   docker stop 'psql_integration'
 fi
 
-docker run --rm -dp 5432:5432 --name 'psql_integration' -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6
+docker run --rm -dp 5432:5432 --name 'psql_integration' -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.8
 sleep 15
 
-docker run --rm --net host postgres:14.6 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres
+docker run --rm --net host postgres:14.8 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres
-docker run --rm --net host postgres:14.6 psql -h localhost -c "create database drydock;" postgres postgres
+docker run --rm --net host postgres:14.8 psql -h localhost -c "create database drydock;" postgres postgres
 
 export DRYDOCK_DB_URL="postgresql+psycopg2://drydock:drydock@localhost:5432/drydock"
 

tools/drydock_image_run.sh

@@ -11,11 +11,11 @@ function start_db {
       sudo docker stop "${PSQL_CONTAINER_NAME}"
     fi
 
-    docker run --rm -dp 5432:5432 --name "${PSQL_CONTAINER_NAME}" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6
+    docker run --rm -dp 5432:5432 --name "${PSQL_CONTAINER_NAME}" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.8
     sleep 15
 
-    docker run --rm --net host postgres:14.6 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres
+    docker run --rm --net host postgres:14.8 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres
-    docker run --rm --net host postgres:14.6 psql -h localhost -c "create database drydock;" postgres postgres
+    docker run --rm --net host postgres:14.8 psql -h localhost -c "create database drydock;" postgres postgres
 }
 
 function customize_conf {

tools/helm_install.sh

@@ -17,7 +17,7 @@
 set -x
 
 HELM=$1
-HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz"}
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"}
 
 
 function install_helm_binary {

tox.ini

@@ -8,6 +8,7 @@ setenv = YAMLDIR = {toxinidir}/python/tests/yaml_samples/
 passenv = http_proxy,HTTP_PROXY,https_proxy,HTTPS_PROXY,no_proxy,NO_PROXY
 deps=
   -r{toxinidir}/python/requirements-frozen.txt
+  -r{toxinidir}/python/test-requirements.txt
 
 [testenv:venv]
 commands = {posargs}
@@ -19,26 +20,11 @@ allowlist_externals=
   sh
 deps=
   -r{toxinidir}/python/requirements-direct.txt
-  -r{toxinidir}/python/test-requirements.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
 commands=
   rm -f python/requirements-frozen.txt
   sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen.txt"
 
-
-[testenv:freeze-bionic]
-basepython=python3
-recreate = True
-allowlist_externals=
-  rm
-  sh
-deps=
-  -r{toxinidir}/python/requirements-direct.txt
-commands=
-  rm -f python/requirements-frozen-bionic.txt
-  sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen-bionic.txt"
-
-
-
 [testenv:yapf]
 allowlist_externals=find
 commands=
@@ -50,6 +36,8 @@ commands=
 
 [testenv:py38]
 usedevelop=True
+allowlist_externals=
+  py.test
 setenv=
   PYTHONWARNING=all
   YAMLDIR={toxinidir}/python/tests/yaml_samples/
@@ -58,21 +46,6 @@ commands=
     --cov=drydock_provisioner \
     {toxinidir}/python/tests/unit/{posargs}
 
-[testenv:py36]
-usedevelop=True
-setenv=
-  PYTHONWARNING=all
-  YAMLDIR={toxinidir}/python/tests/yaml_samples/
-deps=
-  -r{toxinidir}/python/requirements-frozen-bionic.txt
-  -r{toxinidir}/python/test-requirements.txt
-  tox
-commands=
-  py.test \
-    --cov=drydock_provisioner \
-    {toxinidir}/python/tests/unit/{posargs}
-
-
 [testenv:integration]
 passenv=DOCKER_REGISTRY,IMAGE_NAME,IMAGE_PREFIX,IMAGE_TAG,DISTRO
 setenv=
@@ -81,6 +54,7 @@ setenv=
 allowlist_externals=
   sh
   pip3
+  py.test
 commands=
   pip3 install ./python/ --use-pep517
   sh -c {toxinidir}/python/tests/postgres/start_postgres.sh