Rollback apache-airflow back to 1.10.5

This PS is created to :
- roll back apache-airflow back to 1.10.5
- change default chart values from bionic to ubuntu_focal
- save freshly generated config and policy files samples
- in focal Dockerfile postgres client version is pubped up to v15
- change airflow docker image requirements from limited to fully frozen for shipyard-airflow project
- adjusted requirements-direct.txt for shipyard_airflow
- adjusted requirements-direct.txt for shipyard_client
- regenerated requirements-frozen for both projects
- fixed unit tests after upgrading click module
- gen_all tox profile processing has been moved over to py8 gate because it requires focal zuul node to run
- upgraded airskiff gate playbook to include latest treasuremap patchset with updated airskiff site admada manifests

Change-Id: I47e44f5cfa19b2649697e7cc5a31557a6f4fcfea

charts/shipyard/values.yaml

@@ -30,15 +30,15 @@ labels:
 
 images:
   tags:
-    airflow: quay.io/airshipit/airflow:latest-ubuntu_bionic
+    airflow: quay.io/airshipit/airflow:latest-ubuntu_focal
-    shipyard: quay.io/airshipit/shipyard:latest-ubuntu_bionic
+    shipyard: quay.io/airshipit/shipyard:latest-ubuntu_focal
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
     shipyard_db_init: docker.io/postgres:14.5
     shipyard_db_auxiliary: docker.io/postgres:14.5
-    shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_bionic
+    shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_focal
     airflow_db_init: docker.io/postgres:14.5
     rabbit_init: docker.io/rabbitmq:3.7-management
-    airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_bionic
+    airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_focal
     ks_user: docker.io/openstackhelm/heat:ocata
     ks_service: docker.io/openstackhelm/heat:ocata
     ks_endpoints: docker.io/openstackhelm/heat:ocata

doc/requirements.txt

@@ -1,6 +1,6 @@
 #
 # Requirements for creating documentation only.
 #
-Sphinx
+Sphinx<=6.2.1
 sphinx-rtd-theme==0.5.0
 Jinja2<=3.0.3

doc/source/_static/shipyard.conf.sample

@@ -180,9 +180,13 @@
 # and will be removed in the S  release.
 #auth_uri = <None>
 
-# API version of the admin Identity API endpoint. (string value)
+# API version of the Identity API endpoint. (string value)
 #auth_version = <None>
 
+# Interface to use for the Identity API endpoint. Valid values are "public",
+# "internal" (default) or "admin". (string value)
+#interface = internal
+
 # Do not handle authorization requests within the middleware, but delegate the
 # authorization decision to downstream WSGI components. (boolean value)
 #delay_auth_decision = false
@@ -217,14 +221,6 @@
 # The region in which the identity server can be found. (string value)
 #region_name = <None>
 
-# DEPRECATED: Directory used to cache files related to PKI tokens. This option
-# has been deprecated in the Ocata release and will be removed in the P
-# release. (string value)
-# This option is deprecated for removal since Ocata.
-# Its value may be silently ignored in the future.
-# Reason: PKI token format is no longer supported.
-#signing_dir = <None>
-
 # Optionally specify a list of memcached server(s) to use for caching. If left
 # undefined, tokens will instead be cached in-process. (list value)
 # Deprecated group/name - [keystone_authtoken]/memcache_servers
@@ -235,16 +231,6 @@
 # to -1 to disable caching completely. (integer value)
 #token_cache_time = 300
 
-# DEPRECATED: Determines the frequency at which the list of revoked tokens is
-# retrieved from the Identity service (in seconds). A high number of revocation
-# events combined with a low cache duration may significantly reduce
-# performance. Only valid for PKI tokens. This option has been deprecated in
-# the Ocata release and will be removed in the P release. (integer value)
-# This option is deprecated for removal since Ocata.
-# Its value may be silently ignored in the future.
-# Reason: PKI token format is no longer supported.
-#revocation_cache_time = 10
-
 # (Optional) If defined, indicate whether token data should be authenticated or
 # authenticated and encrypted. If MAC, token data is authenticated (with HMAC)
 # in the cache. If ENCRYPT, token data is encrypted and authenticated in the
@@ -280,9 +266,9 @@
 # client connection from the pool. (integer value)
 #memcache_pool_conn_get_timeout = 10
 
-# (Optional) Use the advanced (eventlet safe) memcached client pool. The
+# (Optional) Use the advanced (eventlet safe) memcached client pool. (boolean
-# advanced pool will only work under python 2.x. (boolean value)
+# value)
-#memcache_use_advanced_pool = false
+#memcache_use_advanced_pool = true
 
 # (Optional) Indicate whether to set the X-Service-Catalog header. If False,
 # middleware will not ask for service catalog on token validation and will not
@@ -298,27 +284,6 @@
 # (string value)
 #enforce_token_bind = permissive
 
-# DEPRECATED: If true, the revocation list will be checked for cached tokens.
-# This requires that PKI tokens are configured on the identity server. (boolean
-# value)
-# This option is deprecated for removal since Ocata.
-# Its value may be silently ignored in the future.
-# Reason: PKI token format is no longer supported.
-#check_revocations_for_cached = false
-
-# DEPRECATED: Hash algorithms to use for hashing PKI tokens. This may be a
-# single algorithm or multiple. The algorithms are those supported by Python
-# standard hashlib.new(). The hashes will be tried in the order given, so put
-# the preferred one first for performance. The result of the first hash will be
-# stored in the cache. This will typically be set to multiple values only while
-# migrating from a less secure algorithm to a more secure one. Once all the old
-# tokens are expired this option should be set to a single value for better
-# performance. (list value)
-# This option is deprecated for removal since Ocata.
-# Its value may be silently ignored in the future.
-# Reason: PKI token format is no longer supported.
-#hash_algorithms = md5
-
 # A choice of roles that must be present in a service token. Service tokens are
 # allowed to request that an expired token can be used and so this check should
 # tightly control that only actual services should be sending this token. Roles
@@ -333,6 +298,10 @@
 # possible. (boolean value)
 #service_token_roles_required = false
 
+# The name or type of the service as it appears in the service catalog. This is
+# used to validate tokens that have restricted access rules. (string value)
+#service_type = <None>
+
 # Authentication type to load (string value)
 # Deprecated group/name - [keystone_authtoken]/auth_plugin
 #auth_type = <None>
@@ -472,4 +441,4 @@
 # Warning - Print a Warning level message if the validation fails
 # Error - Return an error when the validation fails and prevent the commit from
 # proceeding
-#deployment_version_commit = Skip
+#deployment_version_commit = Skip

doc/source/_static/shipyard.policy.yaml.sample

@@ -106,3 +106,4 @@
 # targeted release
 # POST  /api/v1.0/actions
 #"workflow_orchestrator:action_test_site": "rule:admin_required"
+

doc/source/client-user-guide.rst

@@ -184,7 +184,7 @@ command. In this example we will execute the ``get actions`` command::
   -e 'OS_PASSWORD=password' -e 'OS_PROJECT_DOMAIN_NAME=default' \
   -e 'OS_PROJECT_NAME=service' -e 'OS_USERNAME=shipyard' \
   -e 'OS_USER_DOMAIN_NAME=default' -e 'OS_IDENTITY_API_VERSION=3' \
-  --rm --net=host airshipit/shipyard:latest-ubuntu_bionic get actions
+  --rm --net=host airshipit/shipyard:latest-ubuntu_focal get actions
 
 The output will resemble the following::
 

doc/source/multi-distro-support.rst

@@ -45,7 +45,7 @@ Following parameters need to be passed as environment/shell variable to make
 command:
 
 DISTRO
-  parameter to identify distro specific Dockerfile, ubuntu_bionic (Default)
+  parameter to identify distro specific Dockerfile, ubuntu_focal (Default)
 
 DISTRO_BASE_IMAGE
   parameter to use different base image other than what's used in DISTRO

images/airflow/Dockerfile.ubuntu_focal

@@ -43,7 +43,7 @@ ARG AIRFLOW_HOME=/usr/local/airflow
 # Moved celery to images/airflow/requirements.txt as apache-airflow uses a
 # version of celery incompatibile with the version of kombu needed by other
 # Airship components
-ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs]==1.10.15"
+ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5"
 ARG DEBIAN_FRONTEND=noninteractive
 ARG ctx_base=src/bin
 
@@ -53,30 +53,39 @@ ARG KUBECTL_VERSION=1.26.3
 # Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
 ENV AIRFLOW_HOME=${AIRFLOW_HOME}
 
-RUN set -ex && \
+RUN set -ex \
-    apt-get -qq update && \
+    && apt-get update \
-    apt-get -y install \
+    && apt-get install -y wget curl \
-    automake \
+        apt-transport-https ca-certificates gnupg \
-    ca-certificates \
+    && echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
-    curl \
+    && curl -o /etc/apt/trusted.gpg.d/postgresql_release_signing_key.asc 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' \
-    git \
+    && apt update \
-    g++ \
+    && apt -y install \
-    libkrb5-dev \
+        automake \
-    libffi-dev \
+        build-essential \
-    libssl-dev \
+        ca-certificates \
-    libpq-dev \
+        curl \
-    libtool \
+        git \
-    libsasl2-dev \
+        g++ \
-    locales \
+        libkrb5-dev \
-    netcat \
+        libffi-dev \
-    netbase \
+        libssl-dev \
-    python3 \
+        libpq-dev \
-    python3-setuptools \
+        libpq5 \
-    python3-pip \
+        libtool \
-    python3-dev \
+        libsasl2-dev \
-    python3-dateutil \
+        locales \
-    make \
+        netcat \
-    --no-install-recommends \
+        netbase \
+        postgresql-client \
+        postgresql-common \
+        python3 \
+        python3-setuptools \
+        python3-pip \
+        python3-dev \
+        python3-dateutil \
+        make \
+        --no-install-recommends \
     && python3 -m pip install -U pip \
     && apt-get clean \
     && rm -rf \
@@ -102,6 +111,7 @@ RUN set -ex \
     && cd .. \
     && rm -fr libyaml
 
+
 # Things that change mostly infrequently
 RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
     && curl -L -o /usr/local/bin/kubectl \
@@ -114,11 +124,11 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
 # Install Airflow directly to allow overriding source
 # COPY images/airflow/requirements.txt /tmp/
 COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/requirements.txt
-RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
+RUN pip3 install -r /tmp/requirements.txt --no-cache-dir
-    && pip3 install $AIRFLOW_SRC --no-cache-dir \
+    # && pip3 install $AIRFLOW_SRC --no-cache-dir
-    && (pip3 uninstall -y snakebite || true) \
+    # && (pip3 uninstall -y snakebite || true) \
-    && (pip3 uninstall -y psycopg2 || true) \
+    # && (pip3 uninstall -y psycopg2 || true) \
-    && (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true)
+    # && (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true)
 
 # Copy scripts used in the container:
 COPY images/airflow/script/*.sh ${AIRFLOW_HOME}/
@@ -143,7 +153,7 @@ RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
 
 COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
 RUN cd /tmp/shipyard \
-    && pip3 install $(pwd)
+    && pip3 install $(pwd) --use-pep517
 
 # Note: The value for the dags and plugins directories that are sourced
 # from the values.yaml of the Shipyard Helm chart need to align with these

images/airflow/script/airflow_start_service.sh

@@ -41,9 +41,9 @@ elif [[ $cmd == *scheduler* ]]; then
 elif [[ $cmd == 'quicktest' ]]; then
     ${python3_path} ${airflow_path} initdb
     ${python3_path} ${airflow_path} webserver -p 8080 &
-    airflow tasks run example_bash_operator runme_0 2018-01-01
+    airflow run example_bash_operator runme_0 2018-01-01
     airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
-    airflow dags state example_bash_operator 2018-01-01
+    airflow dag_state example_bash_operator 2018-01-01
 else
      echo "Invalid Command!"
      exit 1

images/shipyard/Dockerfile.ubuntu_focal

@@ -112,10 +112,10 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
     && python3 -m pip install -U pip \
     && pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \
     && cd /home/shipyard/shipyard_client \
-    && pip3 install $(pwd) --use-pep517 \
+    && python3 setup.py install \
     && pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \
     && cd /home/shipyard/shipyard \
-    && pip3 install $(pwd) --use-pep517 \
+    && python3 setup.py install \
     && apt-get purge -y --auto-remove $buildDeps \
     && apt-get autoremove -yqq --purge \
     && apt-get clean \
@@ -125,7 +125,9 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
         /var/tmp/* \
         /usr/share/man \
         /usr/share/doc \
-        /usr/share/doc-base
+        /usr/share/doc-base \
+    && python3 -m pip install -U pip
+
 
 # Entrypoint
 ENTRYPOINT ["/home/shipyard/entrypoint.sh"]

src/bin/shipyard_airflow/requirements-direct.txt

@@ -1,15 +1,22 @@
+alembic==1.0.1
+apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5
 arrow<=0.17.0
-# apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.15
+celery==4.3
-apache-airflow[crypto,celery,hive,hdfs]==1.10.15
+Flask==1.1.0
+Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@airflow_v1.13.2
+pytz==2018.5
 coverage==5.3
 falcon
 jsonschema<=3.2.0
 keystoneauth1<=5.1.1
+kombu
 kubernetes==26.1.0
+marshmallow-sqlalchemy==0.18.0
 networkx
 oslo.config<=8.7.1
 oslo.policy<=3.10.1
 oslo.utils<=4.12.3
+psycopg2-binary==2.8.4
 pyarrow==6.0.1
 pylibyaml==0.1.0
 PyYAML<=5.4.1
@@ -17,12 +24,16 @@ regex==2020.11.13
 requests==2.23.0
 responses==0.12.1
 setuptools<=45.2.0
-SQLAlchemy<=1.3.20
+SQLAlchemy==1.3.15
+tabulate==0.8.03
 ulid==1.1
 uWSGI==2.0.21
-Werkzeug
+Werkzeug<=0.16.1
 WTForms<=2.3.3
 
+
+itsdangerous==2.0.1
+
 git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand
 git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#egg=drydock_provisioner&subdirectory=python
 git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4#egg=armada

src/bin/shipyard_airflow/requirements-frozen.txt

@@ -1,23 +1,21 @@
 alabaster==0.7.13
-alembic==1.4.3
+alembic==1.0.1
 amqp==2.6.1
-apache-airflow==1.10.15
+apache-airflow==1.10.5
 apispec==1.3.3
-argcomplete==1.12.3
+argcomplete==3.0.8
-Armada @ git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4#egg=armada
+Armada @ git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4
 arrow==0.17.0
-attrs==20.3.0
+attrs==23.1.0
 autopage==0.5.1
 Babel==2.12.1
-bandit==1.6.0
 bcrypt==4.0.1
 Beaker==1.12.0
 billiard==3.6.4.0
 cached-property==1.5.2
 cachetools==5.3.0
-cattrs==1.10.0
+celery==4.3.0
-celery==4.4.7
+certifi==2023.5.7
-certifi==2022.12.7
 cffi==1.15.1
 chardet==3.0.4
 charset-normalizer==3.1.0
@@ -31,18 +29,18 @@ coverage==5.3
 croniter==0.3.37
 cryptography==3.4.8
 debtcollector==2.5.0
-Deckhand @ git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand
+Deckhand @ git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a
 decorator==5.1.1
 deepdiff==5.8.1
 defusedxml==0.7.1
-dill==0.3.6
+dill==0.2.9
 distlib==0.3.6
 dnspython==2.3.0
 docutils==0.17
 dogpile.cache==1.2.0
-drydock-provisioner @ git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#egg=drydock_provisioner&subdirectory=python
+drydock-provisioner @ git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#subdirectory=python
-dulwich==0.21.3
+dulwich==0.21.5
-email-validator==1.3.1
+dumb-init==1.2.5.post1
 eventlet==0.33.3
 extras==1.0.0
 falcon==3.1.1
@@ -50,27 +48,26 @@ fasteners==0.18
 filelock==3.12.0
 fixtures==3.0.0
 flake8==3.8.4
-Flask==1.1.4
+Flask==1.1.0
-Flask-Admin==1.5.4
+Flask-Admin==1.5.3
-Flask-AppBuilder==2.3.4
+Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@b71789d85b632935eca79c0b53fb77f20bf17fe6
-Flask-Babel==1.0.0
+Flask-Babel==0.12.2
 Flask-Caching==1.3.3
 Flask-JWT-Extended==3.25.1
 Flask-Login==0.4.1
 Flask-OpenID==1.3.0
 Flask-SQLAlchemy==2.5.1
-flask-swagger==0.2.14
+flask-swagger==0.2.13
 Flask-WTF==0.14.3
 flower==0.9.7
-funcsigs==1.0.2
+funcsigs==1.0.0
-future==0.18.3
+future==0.16.0
 futurist==2.4.1
 gitdb==4.0.10
 GitPython==3.1.31
 google-auth==2.17.3
-graphviz==0.20.1
 greenlet==2.0.2
-gunicorn==20.1.0
+gunicorn==19.10.0
 hacking==4.1.0
 hmsclient==0.1.1
 html5lib==0.9999999
@@ -79,10 +76,11 @@ humanize==4.6.0
 idna==2.10
 imagesize==1.4.1
 importlib-metadata==2.1.3
-importlib-resources==1.5.0
 iso8601==1.1.0
-itsdangerous==1.1.0
+itsdangerous==2.0.1
-Jinja2==2.11.3
+JayDeBeApi==1.2.3
+Jinja2==2.10.3
+JPype1==1.4.1
 json-merge-patch==0.2
 jsonpath-ng==1.5.3
 jsonpath-rw==1.4.0
@@ -91,19 +89,18 @@ jsonpickle==1.4.1
 jsonschema==3.2.0
 keystoneauth1==5.1.1
 keystonemiddleware==10.2.0
-kombu==4.6.11
+kombu==4.6.3
 kubernetes==26.1.0
-lazy-object-proxy==1.4.3
+lazy-object-proxy==1.9.0
 lockfile==0.12.2
 Mako==1.2.4
 Markdown==2.6.11
 MarkupSafe==2.0.1
-marshmallow==2.21.0
+marshmallow==2.19.5
 marshmallow-enum==1.5.1
-marshmallow-sqlalchemy==0.23.1
+marshmallow-sqlalchemy==0.18.0
 mccabe==0.6.1
 mock==5.0.2
-more-itertools==9.1.0
 msgpack==1.0.5
 natsort==8.3.1
 netaddr==0.8.0
@@ -113,6 +110,7 @@ nose==1.3.7
 numpy==1.24.3
 oauthlib==3.2.2
 ordered-set==4.1.0
+ordereddict==1.1
 os-service-types==1.7.0
 oslo.cache==2.10.1
 oslo.concurrency==5.1.1
@@ -121,32 +119,29 @@ oslo.context==4.1.0
 oslo.db==10.0.0
 oslo.i18n==6.0.0
 oslo.log==4.6.0
-oslo.messaging==12.13.0
+oslo.messaging==10.3.0
-oslo.metrics==0.6.0
 oslo.middleware==4.4.0
 oslo.policy==3.10.1
 oslo.serialization==4.2.0
 oslo.service==3.1.1
 oslo.utils==4.12.3
 packaging==21.3
-pandas==1.5.3
+pandas==0.25.3
 Paste==3.5.0
 PasteDeploy==3.0.1
 PasteScript==3.3.0
 pbr==5.5.1
 pendulum==1.4.4
 pip==23.0.1
-platformdirs==3.4.0
+platformdirs==3.5.0
-pluggy==0.6.0
 ply==3.11
 prettytable==3.7.0
-prison==0.2.1
+prison==0.1.0
-promenade @ git+https://opendev.org/airship/promenade.git@5602e85fb936eeafdc3e4699fd3afbe1576078bc#egg=promenade
+promenade @ git+https://opendev.org/airship/promenade.git@5602e85fb936eeafdc3e4699fd3afbe1576078bc
 prometheus-client==0.8.0
-protobuf==4.22.3
+protobuf==4.22.4
 psutil==5.9.5
-psycopg2-binary==2.9.6
+psycopg2-binary==2.8.4
-pure-sasl==0.6.2
 py==1.11.0
 pyarrow==6.0.1
 pyasn1==0.5.0
@@ -164,20 +159,16 @@ pyparsing==2.4.7
 pyperclip==1.8.2
 pyproject_api==1.5.0
 pyrsistent==0.19.3
-pytest==3.5.0
-pytest-cov==2.5.1
 python-barbicanclient==5.2.0
-python-daemon==2.3.2
+python-daemon==2.1.2
 python-dateutil==2.8.1
 python-editor==1.0.4
 python-keystoneclient==5.1.0
 python-memcached==1.59
 python-mimeparse==1.6.0
-python-nvd3==0.15.0
-python-slugify==4.0.1
 python-subunit==1.4.0
 python3-openid==3.2.0
-pytz==2023.3
+pytz==2018.5
 pytzdata==2020.1
 PyYAML==5.4.1
 regex==2020.11.13
@@ -191,14 +182,13 @@ retry==0.9.2
 rfc3986==2.0.0
 Routes==2.5.1
 rsa==4.9
-sasl==0.3.1
 selector==0.10.1
 setproctitle==1.3.2
 setuptools==45.2.0
 simplejson==3.19.1
 six==1.16.0
 smmap==5.0.0
-snakebite-py3==3.0.5
+snakebite==2.11.0
 snowballstemmer==2.2.0
 Sphinx==3.3.1
 sphinx-rtd-theme==0.5.0
@@ -208,30 +198,25 @@ sphinxcontrib-htmlhelp==2.0.1
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
 sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==1.3.20
+SQLAlchemy==1.3.15
-SQLAlchemy-JSONField==0.9.0
 sqlalchemy-migrate==0.13.0
-SQLAlchemy-Utils==0.41.0
 sqlparse==0.4.4
 statsd==4.0.1
 stevedore==5.0.0
-tabulate==0.8.10
+tabulate==0.8.3
 Tempita==0.5.2
 tenacity==4.12.0
-testfixtures==7.1.0
+termcolor==1.1.0
 testrepository==0.0.20
 testresources==2.0.1
 testscenarios==0.5.0
 testtools==2.5.0
-text-unidecode==1.3
+text-unidecode==1.2
 thrift==0.16.0
-thrift-sasl==0.4.3
 tiddlyweb==2.4.3
-toml==0.10.2
 tomli==2.0.1
-tomlkit==0.11.7
+tomlkit==0.11.8
 tornado==5.1.1
-tox==3.12.1
 typing-extensions==3.7.2
 tzlocal==1.5.1
 ulid==1.1
@@ -239,7 +224,7 @@ unicodecsv==0.14.1
 urllib3==1.25.11
 uWSGI==2.0.21
 vine==1.3.0
-virtualenv==20.22.0
+virtualenv==20.23.0
 wcwidth==0.2.6
 WebOb==1.8.7
 websocket-client==1.5.1

src/bin/shipyard_airflow/test-requirements.txt

@@ -5,4 +5,11 @@ pytest-cov==2.5.1
 testfixtures
 tox<=4.0.0
 typing-extensions==3.7.2
-pyflakes>=2.1.1
+pyflakes>=2.1.1
+# Pin apache-airflow 1.10.5 to align with version in airflow images requirements.txt
+apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5
+# install postgres seperately as apache-airflow 1.10.5 forces postgres to use
+# psycopg2 instead of psycopg2-binary which requires additional apt packages
+# to be installed, i.e. postgres-devel. Otherwise the following error is seen:
+# Error: pg_config executable not found. Only seems to be an issue in xenial
+postgres>=3.0.0

src/bin/shipyard_airflow/tox.ini

@@ -7,7 +7,9 @@ setenv=
   PYTHONWARNING=all
   LANG=C.UTF-8
   LC_ALL=C.UTF-8
-deps = -r{toxinidir}/requirements-frozen.txt
+deps =
+  -r{toxinidir}/requirements-frozen.txt
+  -r{toxinidir}/test-requirements.txt
 
 
 [testenv:freeze]
@@ -17,7 +19,6 @@ allowlist_externals=
   sh
 deps=
   -r{toxinidir}/requirements-direct.txt
-  -r{toxinidir}/test-requirements.txt
 commands=
   rm -f {toxinidir}/requirements-frozen.txt
   sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
@@ -27,9 +28,16 @@ commands=
 skipsdist=True
 setenv=
   SLUGIFY_USES_TEXT_UNIDECODE=yes
+allowlist_externals=
+  bash
+  airflow
 commands =
+  bash -c "rm -f ${HOME}/airflow/airflow.db"
   pip install -r{toxinidir}/test-requirements.txt
   airflow initdb
+  airflow run example_bash_operator runme_0 2018-01-01
+  airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
+  airflow dag_state example_bash_operator 2018-01-01
   pytest \
     {posargs} \
     --cov-branch \
@@ -74,6 +82,7 @@ commands =
 
 [testenv:gen_all]
 # Combined to make these run together instead of setting up separately
+skipsdist=True
 setenv=
   SLUGIFY_USES_TEXT_UNIDECODE=yes
 deps =

src/bin/shipyard_client/requirements-direct.txt

@@ -15,10 +15,19 @@
 # CLI/Client requirements
 
 arrow<=0.17.0
-click==6.7
+click==7.1.2
 click-default-group==1.2
 keystoneauth1<=5.1.1
 requests==2.23.0
 PTable==0.9.2
 pylibyaml==0.1.0
 PyYAML<=5.4.1
+Flask==1.1.0
+coverage==5.3
+itsdangerous==2.0.1
+Jinja2==2.10.3
+MarkupSafe==2.0.1
+pbr==5.5.1
+python-dateutil==2.8.1
+setuptools<=45.2.0
+Werkzeug==0.16.1

src/bin/shipyard_client/requirements-frozen.txt

@@ -1,40 +1,27 @@
-amqp==2.6.1
 arrow==0.17.0
-attrs==23.1.0
+certifi==2023.5.7
-bandit==1.6.0
-certifi==2022.12.7
 chardet==3.0.4
-click==6.7
+click==7.1.2
 click-default-group==1.2
-coverage==7.2.3
+coverage==5.3
-flake8==3.8.4
+Flask==1.1.0
-gitdb==4.0.10
-GitPython==3.1.31
 idna==2.10
 iso8601==1.1.0
+itsdangerous==2.0.1
+Jinja2==2.10.3
 keystoneauth1==5.1.1
-mccabe==0.6.1
+MarkupSafe==2.0.1
-more-itertools==9.1.0
 os-service-types==1.7.0
-pbr==5.11.1
+pbr==5.5.1
 pip==23.0.1
-pluggy==0.6.0
 PTable==0.9.2
-py==1.11.0
-pycodestyle==2.6.0
-pyflakes==2.2.0
 pylibyaml==0.1.0
-pytest==3.5.0
+python-dateutil==2.8.1
-pytest-cov==2.5.1
-python-dateutil==2.8.2
 PyYAML==5.4.1
 requests==2.23.0
-responses==0.10.2
+setuptools==45.2.0
-setuptools==67.6.0
 six==1.16.0
-smmap==5.0.0
 stevedore==5.0.0
-testfixtures==5.1.1
 urllib3==1.25.11
-vine==1.3.0
+Werkzeug==0.16.1
 wheel==0.40.0

src/bin/shipyard_client/tests/unit/cli/create/test_create_commands.py

@@ -187,7 +187,7 @@ def test_create_configdocs_directory_empty():
             '--directory=' + dir1
         ])
     _method.assert_not_called()
-    assert b'directory does not contain any YAML files' in result.output_bytes
+    assert b'directory does not contain any YAML files' in result.stdout_bytes
 
 
 def test_create_configdocs_multi_directory():

src/bin/shipyard_client/tox.ini

@@ -17,7 +17,6 @@ allowlist_externals=
   sh
 deps=
   -r{toxinidir}/requirements-direct.txt
-  -r{toxinidir}/test-requirements.txt
 commands=
   rm -f {toxinidir}/requirements-frozen.txt
   sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"

tools/airflow_image_run.sh

@@ -47,6 +47,7 @@ fi
 docker stop airflow_test
 docker rm airflow_test
 
+
 if [ ${TEST_RESP:(-7)} == "success" ]; then
     exit 0
 else

tools/gate/playbooks/airskiff-deploy.yaml

@@ -22,6 +22,7 @@
     - ensure-docker
 
   tasks:
+
     - name: Clone Required Repositories
       shell: |
         export CLONE_SHIPYARD={{ CLONE_SHIPYARD }}
@@ -45,15 +46,23 @@
     - name: Build Shipyard and Airflow with submitted changes
       shell: |
         make
+        docker pull quay.io/airshipit/shipyard:latest-ubuntu_bionic
+        docker rm quay.io/airshipit/shipyard:latest-ubuntu_bionic
+        docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal quay.io/airshipit/shipyard:latest-ubuntu_bionic
+        docker pull quay.io/airshipit/airflow:latest-ubuntu_bionic
+        docker rm quay.io/airshipit/airflow:latest-ubuntu_bionic
+        docker tag quay.io/airshipit/airflow:latest-ubuntu_focal quay.io/airshipit/airflow:latest-ubuntu_bionic
       args:
         chdir: "{{ zuul.project.src_dir }}"
       become: yes
 
-    - name: Setup clients
+    - name: Build all charts locally
       shell: |
-        ./tools/deployment/airskiff/developer/020-setup-client.sh
+        set -ex
+        ./tools/deployment/airskiff/developer/015-make-all-charts.sh
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
 
     - name: Deploy Airship components using Armada
       shell: |

tools/shipyard.sh

@@ -38,7 +38,7 @@ set -e
 # Source Base Docker Command
 SHIPYARD_HOSTPATH=${SHIPYARD_HOSTPATH:-"/target"}
 NAMESPACE="${NAMESPACE:-ucp}"
-DISTRO="${DISTRO:-ubuntu_bionic}"
+DISTRO="${DISTRO:-ubuntu_focal}"
 SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master-${DISTRO}}"
 # set default value for OS_PASSWORD if it's not set
 # this doesn't actually get exported to environment

tools/shipyard_docker_base_command.sh

@@ -19,7 +19,7 @@
 # before executing this script if they differ from the default values.
 #
 NAMESPACE="${NAMESPACE:-ucp}"
-DISTRO="${DISTRO:-ubuntu_bionic}"
+DISTRO="${DISTRO:-ubuntu_focal}"
 SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master-${DISTRO}}"
 
 # Define Base Docker Command

tox.ini

@@ -23,9 +23,6 @@ allowlist_externals=
     cp
     tox
 commands=
-    tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all
-    cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample
-    cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample
     rm -rf {toxinidir}/doc/build
     sphinx-build -W -b html {toxinidir}/doc/source {toxinidir}/doc/build/html
 deps= -r{toxinidir}/doc/requirements.txt
@@ -40,11 +37,17 @@ commands=
   tox -c {toxinidir}/src/bin/shipyard_client/tox.ini
 
 [testenv:py38]
+deps =
+  tox<=4.0.0
 allowlist_externals=
     tox
+    bash
 commands=
   tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py38
   tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py38
+  tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all
+  bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample"
+  bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample"
 
 [testenv:pep8]
 allowlist_externals=
@@ -52,3 +55,29 @@ allowlist_externals=
 commands=
   tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e pep8
   tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e pep8
+
+
+[testenv:freeze]
+allowlist_externals=
+    tox
+commands=
+  tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e freeze
+  tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e freeze
+
+
+[testenv:shipyard_client]
+deps =
+  -r{toxinidir}/src/bin/shipyard_airflow/requirements-frozen.txt
+  -r{toxinidir}/src/bin/shipyard_client/requirements-frozen.txt
+skipsdist=True
+allowlist_externals=
+    pytest
+commands =
+  pytest \
+    {posargs} \
+    --cov-branch \
+    --cov-report=term-missing:skip-covered \
+    --cov-config=.coveragerc \
+    --cov=shipyard_client \
+    --cov-report=html \
+    -vv