diff --git a/.zuul.yaml b/.zuul.yaml
index aea9a16a..cd126a57 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -14,6 +14,7 @@
templates:
- docs-on-readthedocs
- openstack-python38-jobs
+ - openstack-python36-jobs
vars:
rtd_webhook_id: '38576'
rtd_project_name: 'airship-shipyard'
@@ -21,20 +22,28 @@
check:
jobs:
- openstack-tox-pep8
+ - airship-shipyard-openstack-tox-cover-focal
- airship-shipyard-chart-build-gate
- airship-shipyard-chart-build-latest-htk
- airship-shipyard-whitespace-lint-gate
- - airship-shipyard-airskiff-deployment
+ - airship-shipyard-airskiff-deployment-bionic
+ - airship-shipyard-airskiff-deployment-focal
- airship-shipyard-image-gate-ubuntu_focal
+ - airship-shipyard-image-gate-ubuntu_bionic
gate:
jobs:
- openstack-tox-pep8
+ - airship-shipyard-openstack-tox-cover-focal
- airship-shipyard-chart-build-gate
- airship-shipyard-whitespace-lint-gate
+ - airship-shipyard-airskiff-deployment-bionic
+ - airship-shipyard-airskiff-deployment-focal
- airship-shipyard-image-gate-ubuntu_focal
+ - airship-shipyard-image-gate-ubuntu_bionic
post:
jobs:
- - airship-shipyard-docker-build-post-ubuntu_focal
+ - airship-shipyard-docker-build-publish-ubuntu_focal
+ - airship-shipyard-docker-build-publish-ubuntu_bionic
- shipyard-upload-git-mirror
- nodeset:
@@ -49,12 +58,21 @@
- name: primary
label: ubuntu-focal
+
+- job:
+ name: airship-shipyard-openstack-tox-cover-focal
+ parent: openstack-tox-cover
+ description: Runs cover job on focal
+ nodeset: airship-shipyard-single-node-focal
+
- job:
name: airship-shipyard-chart-build-gate
description: Build charts using pinned Helm toolkit.
timeout: 900
run: tools/gate/playbooks/build-charts.yaml
nodeset: airship-shipyard-single-node-focal
+ vars:
+ HTK_COMMIT: a7cd689280cdbc0acd04a7a1b745941260e8700b
- job:
name: airship-shipyard-chart-build-latest-htk
@@ -78,13 +96,14 @@
description: |
Base job for running airship-shipyard image related jobs.
timeout: 3600
- run: tools/gate/playbooks/run-image.yaml
+ run: tools/gate/playbooks/docker-image-build.yaml
nodeset: airship-shipyard-single-node-focal
- irrelevant-files:
+ irrelevant-files: &irrelevant-files
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
+
- job:
name: airship-shipyard-image-gate-ubuntu_focal
description: |
@@ -98,7 +117,20 @@
patch_set: true
- job:
- name: airship-shipyard-airskiff-deployment
+ name: airship-shipyard-image-gate-ubuntu_bionic
+ description: |
+ Run shipyard-image build for ubuntu_bionic
+ parent: airship-shipyard-image-base
+ irrelevant-files: *irrelevant-files
+ vars:
+ publish: false
+ distro: ubuntu_bionic
+ tags:
+ dynamic:
+ patch_set: true
+
+- job:
+ name: airship-shipyard-airskiff-deployment-focal
nodeset: airship-shipyard-single-node-focal
description: |
Deploy Memcached using Airskiff and submitted Armada changes.
@@ -114,29 +146,35 @@
vars:
CLONE_SHIPYARD: false
HTK_COMMIT: master
- irrelevant-files: &irrelevant-files
- - ^.*\.rst$
- - ^doc/.*$
- - ^releasenotes/.*$
+ SHIPYARD_IMAGE_DISTRO: ubuntu_focal
+ irrelevant-files: *irrelevant-files
- job:
- name: airship-shipyard-image-gate-opensuse
+ name: airship-shipyard-airskiff-deployment-bionic
+ nodeset: airship-shipyard-single-node-focal
description: |
- Run shipyard-image build for opensuse.
- parent: airship-shipyard-image-base
+ Deploy Memcached using Airskiff and submitted Armada changes.
+ timeout: 9600
+ voting: true
+ pre-run:
+ - tools/gate/playbooks/airskiff-reduce-site.yaml
+ - tools/gate/playbooks/git-config.yaml
+ run: tools/gate/playbooks/airskiff-deploy.yaml
+ post-run: tools/gate/playbooks/debug-report.yaml
+ required-projects:
+ - airship/treasuremap
vars:
- publish: false
- distro: opensuse_15
- tags:
- dynamic:
- patch_set: true
+ CLONE_SHIPYARD: false
+ HTK_COMMIT: master
+ SHIPYARD_IMAGE_DISTRO: ubuntu_bionic
+ irrelevant-files: *irrelevant-files
- job:
- name: airship-shipyard-docker-build-post-ubuntu_focal
+ name: airship-shipyard-docker-build-publish-ubuntu_focal
timeout: 1800
run: tools/gate/playbooks/run-image.yaml
- nodeset: airship-shipyard-single-node
+ nodeset: airship-shipyard-single-node-focal
irrelevant-files: *irrelevant-files
secrets:
- airship_shipyard_quay_creds
@@ -151,19 +189,16 @@
- latest
- job:
- name: airship-shipyard-docker-build-post-opensuse
+ name: airship-shipyard-docker-build-publish-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/run-image.yaml
- nodeset: airship-shipyard-single-node
- irrelevant-files:
- - ^.*\.rst$
- - ^doc/.*$
- - ^releasenotes/.*$
+ nodeset: airship-shipyard-single-node-focal
+ irrelevant-files: *irrelevant-files
secrets:
- airship_shipyard_quay_creds
vars:
publish: true
- distro: opensuse_15
+ distro: ubuntu_bionic
tags:
dynamic:
branch: true
@@ -171,6 +206,7 @@
static:
- latest
+
- secret:
name: airship_shipyard_quay_creds
data:
@@ -205,15 +241,15 @@
git_mirror_repository: airshipit/shipyard
secrets:
- name: git_mirror_credentials
- secret: shipyard-airshipit-github-secret
+ secret: shipyard-airshipit-github-secret-2023-03-23
pass-to-parent: true
- secret:
- name: shipyard-airshipit-github-secret
+ name: shipyard-airshipit-github-secret-2023-03-23
data:
user: git
host: github.com
- host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
+ host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
ssh_key: !encrypted/pkcs1-oaep
- hUsE3gOWqb6aoNVSpqRHcUbqi0wOjRJ47viZWr7JopICoyhIerZ6DN4dGRaFjJ+8UtQvD
uXKYHZ35+YbsJDQttIadtVvj5Va8hIToI/rARSktyDp4zb+fK829INKuOjSszowhGoDuv
diff --git a/Makefile b/Makefile
index 9dd33cf0..375a3fc7 100644
--- a/Makefile
+++ b/Makefile
@@ -40,18 +40,36 @@ IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO}
IMAGE_DIR:=images/$(IMAGE_NAME)
.PHONY: images
-#Build all images in the list
-images: $(IMAGE_NAME)
-#Build and run all images in list
-#sudo make images IMAGE_NAME=airflow will Build and Run airflow
-#sudo make images will build and run airflow and shipyard
-$(IMAGE_NAME):
- @echo
- @echo "===== Processing [$@] image ====="
- @make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/$@ IMAGE_NAME=$@
- @make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/$@_image_run.sh
# Build all docker images for this project
+images: build_images
+
+build_images: build_airflow build_shipyard
+
+run_images: build_airflow run_airflow build_shipyard run_shipyard
+
+#Build all images in list
+build_airflow:
+ @echo
+ @echo "===== Processing [airflow] image ====="
+ @make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/airflow:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/airflow IMAGE_NAME=airflow
+
+build_shipyard:
+ @echo
+ @echo "===== Processing [shipyard] image ====="
+ @make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/shipyard:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/shipyard IMAGE_NAME=shipyard
+
+
+#Run all images in list
+
+run_airflow:
+ @echo
+ @echo "===== Processing [airflow] image ====="
+ @make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/airflow:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/airflow_image_run.sh
+run_shipyard:
+ @echo
+ @echo "===== Processing [shipyard] image ====="
+ @make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/shipyard:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/shipyard_image_run.sh
# Create tgz of the chart
.PHONY: charts
diff --git a/images/airflow/Dockerfile.ubuntu_bionic b/images/airflow/Dockerfile.ubuntu_bionic
index 640430fd..b4134a21 100644
--- a/images/airflow/Dockerfile.ubuntu_bionic
+++ b/images/airflow/Dockerfile.ubuntu_bionic
@@ -64,10 +64,13 @@ RUN set -ex && \
libffi-dev \
libssl-dev \
libpq-dev \
+ libpq5 \
libtool \
locales \
netcat \
netbase \
+ postgresql-client \
+ postgresql-common \
python3 \
python3-setuptools \
python3-pip \
@@ -110,7 +113,7 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
# Note - removing snakebite (python 2 vs. 3). See:
# https://github.com/puckel/docker-airflow/issues/77
# Install Airflow directly to allow overriding source
-COPY images/airflow/requirements.txt /tmp/
+COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /tmp/requirements.txt
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
&& pip3 install $AIRFLOW_SRC --no-cache-dir \
&& (pip3 uninstall -y snakebite || true) \
@@ -135,8 +138,8 @@ ENV PBR_VERSION 0.1a1
# Since Shipyard and Airflow are built together as images, this should prevent
# stale or out-of-date code between these parts.
# Shipyard requirements, source and installation
-COPY ${ctx_base}/shipyard_airflow/requirements.txt /tmp/api_requirements.txt
-RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
+# COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /tmp/api_requirements.txt
+# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
RUN cd /tmp/shipyard \
diff --git a/images/airflow/Dockerfile.ubuntu_focal b/images/airflow/Dockerfile.ubuntu_focal
index 5a6bcf5f..42415b9f 100644
--- a/images/airflow/Dockerfile.ubuntu_focal
+++ b/images/airflow/Dockerfile.ubuntu_focal
@@ -148,8 +148,8 @@ ENV PBR_VERSION 0.1a1
# Since Shipyard and Airflow are built together as images, this should prevent
# stale or out-of-date code between these parts.
# Shipyard requirements, source and installation
-COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt
-RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
+# COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt
+# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
RUN cd /tmp/shipyard \
diff --git a/images/shipyard/Dockerfile.ubuntu_bionic b/images/shipyard/Dockerfile.ubuntu_bionic
index e6dfd600..77443cd3 100644
--- a/images/shipyard/Dockerfile.ubuntu_bionic
+++ b/images/shipyard/Dockerfile.ubuntu_bionic
@@ -85,8 +85,8 @@ RUN chown -R shipyard: /home/shipyard \
&& chmod +x /home/shipyard/entrypoint.sh
# Requirements and Shipyard source
-COPY ${ctx_base}/shipyard_airflow/requirements.txt /home/shipyard/api_requirements.txt
-COPY ${ctx_base}/shipyard_client/requirements.txt /home/shipyard/client_requirements.txt
+COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /home/shipyard/api_requirements.txt
+COPY ${ctx_base}/shipyard_client/requirements-frozen-bionic.txt /home/shipyard/client_requirements.txt
COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/
COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
diff --git a/src/bin/shipyard_airflow/requirements-direct.txt b/src/bin/shipyard_airflow/requirements-direct.txt
index 4a2c0b7e..f6c053dd 100644
--- a/src/bin/shipyard_airflow/requirements-direct.txt
+++ b/src/bin/shipyard_airflow/requirements-direct.txt
@@ -34,7 +34,7 @@ WTForms<=2.3.3
itsdangerous==2.0.1
-git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e#egg=deckhand
-git+https://opendev.org/airship/drydock.git@375abedb8aa3a413ca70a47aef467f39d65e1aee#egg=drydock_provisioner&subdirectory=python
-git+https://opendev.org/airship/armada.git@4be7fa44a743fbcdf792e2bdca3af6c95b540f97#egg=armada
-git+https://opendev.org/airship/promenade.git@45bcba068eb42fe6ba54d494b12122600dcb2c6c#egg=promenade
+git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5#egg=deckhand
+git+https://opendev.org/airship/drydock.git@73d22a488f7f5b7217e86e86da9f38b5b9085009#egg=drydock_provisioner&subdirectory=python
+git+https://opendev.org/airship/armada.git@812546c87595b1d8a1bc0da931d644473b03b31f#egg=armada
+git+https://opendev.org/airship/promenade.git@2f2455f0d159287565a279a99c0bbeb235e81ffb#egg=promenade
diff --git a/src/bin/shipyard_airflow/requirements-frozen-bionic.txt b/src/bin/shipyard_airflow/requirements-frozen-bionic.txt
new file mode 100644
index 00000000..ab99742d
--- /dev/null
+++ b/src/bin/shipyard_airflow/requirements-frozen-bionic.txt
@@ -0,0 +1,241 @@
+alabaster==0.7.13
+alembic==1.0.1
+amqp==2.6.1
+apache-airflow==1.10.5
+apispec==1.3.3
+argcomplete==3.0.8
+Armada @ git+https://opendev.org/airship/armada.git@812546c87595b1d8a1bc0da931d644473b03b31f
+arrow==0.17.0
+attrs==22.2.0
+autopage==0.5.1
+Babel==2.11.0
+bcrypt==4.0.1
+Beaker==1.12.0
+billiard==3.6.4.0
+cached-property==1.5.2
+cachetools==4.2.4
+celery==4.3.0
+certifi==2023.5.7
+cffi==1.15.1
+chardet==3.0.4
+charset-normalizer==3.0.1
+click==7.1.2
+cliff==3.10.1
+cmd2==2.4.3
+colorama==0.4.5
+colorlog==4.0.2
+configparser==3.5.3
+coverage==5.3
+croniter==0.3.37
+cryptography==3.4.8
+debtcollector==2.5.0
+Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5
+decorator==4.4.2
+deepdiff==5.7.0
+defusedxml==0.7.1
+dill==0.2.9
+distlib==0.3.6
+dnspython==2.2.1
+docutils==0.17
+dogpile.cache==1.2.1
+drydock-provisioner @ git+https://opendev.org/airship/drydock.git@73d22a488f7f5b7217e86e86da9f38b5b9085009#subdirectory=python
+dulwich==0.20.50
+dumb-init==1.2.5.post1
+eventlet==0.33.3
+extras==1.0.0
+falcon==3.1.1
+fasteners==0.18
+filelock==3.4.1
+fixtures==3.0.0
+flake8==3.8.4
+Flask==1.1.0
+Flask-Admin==1.5.3
+Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@b71789d85b632935eca79c0b53fb77f20bf17fe6
+Flask-Babel==0.12.2
+Flask-Caching==1.3.3
+Flask-JWT-Extended==3.25.1
+Flask-Login==0.4.1
+Flask-OpenID==1.3.0
+Flask-SQLAlchemy==2.5.1
+flask-swagger==0.2.13
+Flask-WTF==0.14.3
+flower==0.9.7
+funcsigs==1.0.0
+future==0.16.0
+futurist==2.4.1
+gitdb==4.0.9
+GitPython==3.1.18
+google-auth==2.19.0
+greenlet==2.0.2
+gunicorn==19.10.0
+hacking==4.1.0
+hmsclient==0.1.1
+html5lib==0.9999999
+httpexceptor==1.4.0
+humanize==3.14.0
+idna==2.10
+imagesize==1.4.1
+importlib-metadata==2.1.3
+importlib-resources==5.4.0
+iso8601==1.1.0
+itsdangerous==2.0.1
+JayDeBeApi==1.2.3
+Jinja2==2.10.3
+JPype1==1.3.0
+json-merge-patch==0.2
+jsonpath-ng==1.5.3
+jsonpath-rw==1.4.0
+jsonpath-rw-ext==1.2.2
+jsonpickle==1.4.1
+jsonschema==3.2.0
+keystoneauth1==5.1.1
+keystonemiddleware==9.5.0
+kombu==4.6.3
+kubernetes==26.1.0
+lazy-object-proxy==1.7.1
+lockfile==0.12.2
+Mako==1.1.6
+Markdown==2.6.11
+MarkupSafe==2.0.1
+marshmallow==2.19.5
+marshmallow-enum==1.5.1
+marshmallow-sqlalchemy==0.18.0
+mccabe==0.6.1
+mock==5.0.2
+msgpack==1.0.5
+natsort==8.2.0
+netaddr==0.8.0
+netifaces==0.11.0
+networkx==2.5.1
+nose==1.3.7
+numpy==1.19.5
+oauthlib==3.2.2
+ordered-set==4.0.2
+ordereddict==1.1
+os-service-types==1.7.0
+oslo.cache==2.10.1
+oslo.concurrency==4.5.1
+oslo.config==8.7.1
+oslo.context==4.1.0
+oslo.db==10.0.0
+oslo.i18n==5.1.0
+oslo.log==4.6.0
+oslo.messaging==10.3.0
+oslo.middleware==4.4.0
+oslo.policy==3.10.1
+oslo.serialization==4.2.0
+oslo.service==2.8.0
+oslo.utils==4.12.3
+packaging==21.3
+pandas==0.25.3
+Paste==3.5.0
+PasteDeploy==2.1.1
+PasteScript==3.3.0
+pbr==5.5.1
+pendulum==1.4.4
+pip==21.3.1
+platformdirs==2.4.0
+ply==3.11
+prettytable==2.5.0
+prison==0.1.0
+promenade @ git+https://opendev.org/airship/promenade.git@2f2455f0d159287565a279a99c0bbeb235e81ffb
+prometheus-client==0.8.0
+protobuf==3.19.6
+psutil==5.9.5
+psycopg2-binary==2.8.4
+py==1.11.0
+pyarrow==6.0.1
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pycadf==3.1.1
+pycodestyle==2.6.0
+pycparser==2.21
+pyflakes==2.2.0
+Pygments==2.14.0
+PyHive==0.6.5
+PyJWT==1.7.1
+pylibyaml==0.1.0
+pymongo==4.1.1
+pyparsing==2.4.7
+pyperclip==1.8.2
+pyproject-api==0.1.0
+pyrsistent==0.18.0
+python-barbicanclient==5.2.0
+python-daemon==2.1.2
+python-dateutil==2.8.1
+python-editor==1.0.4
+python-keystoneclient==4.5.0
+python-memcached==1.59
+python-mimeparse==1.6.0
+python-subunit==1.4.0
+python3-openid==3.2.0
+pytz==2018.5
+pytzdata==2020.1
+PyYAML==5.4.1
+regex==2020.11.13
+reno==4.0.0
+repoze.lru==0.7
+requests==2.23.0
+requests-oauthlib==1.3.1
+resolver==0.2.1
+responses==0.12.1
+retry==0.9.2
+rfc3986==1.5.0
+Routes==2.5.1
+rsa==4.9
+selector==0.10.1
+setproctitle==1.2.3
+setuptools==45.2.0
+simplejson==3.19.1
+six==1.16.0
+smmap==5.0.0
+snakebite==2.11.0
+snowballstemmer==2.2.0
+Sphinx==3.3.1
+sphinx-rtd-theme==0.5.0
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.0
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.5
+SQLAlchemy==1.3.15
+sqlalchemy-migrate==0.13.0
+sqlparse==0.4.4
+statsd==4.0.1
+stevedore==3.5.2
+tabulate==0.8.3
+Tempita==0.5.2
+tenacity==4.12.0
+termcolor==1.1.0
+testrepository==0.0.20
+testresources==2.0.1
+testscenarios==0.5.0
+testtools==2.5.0
+text-unidecode==1.2
+thrift==0.16.0
+tiddlyweb==2.4.3
+tomli==1.2.3
+tomlkit==0.11.6
+tornado==5.1.1
+typing-extensions==3.7.4.3
+tzlocal==1.5.1
+ulid==1.1
+unicodecsv==0.14.1
+urllib3==1.25.11
+uWSGI==2.0.21
+vine==1.3.0
+virtualenv==20.16.2
+wcwidth==0.2.6
+WebOb==1.8.7
+websocket-client==1.3.1
+Werkzeug==0.16.1
+wheel==0.37.1
+wrapt==1.15.0
+wsgi-intercept==1.11.0
+WTForms==2.3.3
+xmltodict==0.13.0
+yappi==1.4.0
+yq==3.2.1
+zipp==3.6.0
+zope.deprecation==4.4.0
diff --git a/src/bin/shipyard_airflow/setup.cfg b/src/bin/shipyard_airflow/setup.cfg
index a934df2e..88fa9f6b 100644
--- a/src/bin/shipyard_airflow/setup.cfg
+++ b/src/bin/shipyard_airflow/setup.cfg
@@ -12,6 +12,7 @@ classifier =
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 3
+ Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.10
diff --git a/src/bin/shipyard_airflow/tox.ini b/src/bin/shipyard_airflow/tox.ini
index 8bc6ad26..1f1639ab 100644
--- a/src/bin/shipyard_airflow/tox.ini
+++ b/src/bin/shipyard_airflow/tox.ini
@@ -12,6 +12,17 @@ deps =
-r{toxinidir}/test-requirements.txt
+[testenv:freeze-bionic]
+recreate = True
+allowlist_externals=
+ rm
+ sh
+deps=
+ -r{toxinidir}/requirements-direct.txt
+commands=
+ rm -f {toxinidir}/requirements-frozen-bionic.txt
+ sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen-bionic.txt"
+
[testenv:freeze]
recreate = True
allowlist_externals=
@@ -28,11 +39,28 @@ commands=
skipsdist=True
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
+basepython=python3.8
allowlist_externals=
bash
airflow
commands =
- bash -c "rm -f ${HOME}/airflow/airflow.db"
+ bash -c "rm -f $HOME/airflow/airflow.db"
+ pip install -r{toxinidir}/test-requirements.txt
+ airflow initdb
+ airflow run example_bash_operator runme_0 2018-01-01
+ airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
+ airflow dag_state example_bash_operator 2018-01-01
+ pytest {posargs} -vv
+
+[testenv:cover]
+skipsdist=True
+setenv=
+ SLUGIFY_USES_TEXT_UNIDECODE=yes
+allowlist_externals=
+ bash
+ airflow
+commands =
+ bash -c "rm -f $HOME/airflow/airflow.db"
pip install -r{toxinidir}/test-requirements.txt
airflow initdb
airflow run example_bash_operator runme_0 2018-01-01
@@ -44,9 +72,31 @@ commands =
--cov-report=term-missing:skip-covered \
--cov-config=.coveragerc \
--cov=shipyard_airflow \
- --cov-report=html \
+ --cov-report html:cover \
+ --cov-report xml:cover/coverage.xml \
+ --cov-report term \
-vv
+[testenv:py36]
+skipsdist=True
+basepython=python3.6
+deps =
+ -r{toxinidir}/requirements-frozen-bionic.txt
+setenv=
+ SLUGIFY_USES_TEXT_UNIDECODE=yes
+allowlist_externals=
+ bash
+ airflow
+commands =
+ bash -c "rm -f $HOME/airflow/airflow.db"
+ pip install -r{toxinidir}/test-requirements.txt
+ airflow initdb
+ airflow run example_bash_operator runme_0 2018-01-01
+ airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
+ airflow dag_state example_bash_operator 2018-01-01
+ pytest {posargs} -vv
+
+
[testenv:pep8]
deps=
flake8>=3.3.0
diff --git a/src/bin/shipyard_client/requirements-frozen-bionic.txt b/src/bin/shipyard_client/requirements-frozen-bionic.txt
new file mode 100644
index 00000000..597a2e7d
--- /dev/null
+++ b/src/bin/shipyard_client/requirements-frozen-bionic.txt
@@ -0,0 +1,30 @@
+arrow==0.17.0
+certifi==2023.5.7
+chardet==3.0.4
+click==7.1.2
+click-default-group==1.2
+coverage==5.3
+Flask==1.1.0
+idna==2.10
+importlib-metadata==4.8.3
+iso8601==1.1.0
+itsdangerous==2.0.1
+Jinja2==2.10.3
+keystoneauth1==5.1.1
+MarkupSafe==2.0.1
+os-service-types==1.7.0
+pbr==5.5.1
+pip==21.3.1
+PTable==0.9.2
+pylibyaml==0.1.0
+python-dateutil==2.8.1
+PyYAML==5.4.1
+requests==2.23.0
+setuptools==45.2.0
+six==1.16.0
+stevedore==3.5.2
+typing_extensions==4.1.1
+urllib3==1.25.11
+Werkzeug==0.16.1
+wheel==0.37.1
+zipp==3.6.0
diff --git a/src/bin/shipyard_client/tox.ini b/src/bin/shipyard_client/tox.ini
index 9cbde55a..46b45c88 100644
--- a/src/bin/shipyard_client/tox.ini
+++ b/src/bin/shipyard_client/tox.ini
@@ -7,8 +7,21 @@ setenv=
PYTHONWARNING=all
LC_ALL=C.UTF-8
LC_ALL=C.UTF-8
-deps = -r{toxinidir}/requirements-frozen.txt
- -r{toxinidir}/test-requirements.txt
+deps =
+ -r{toxinidir}/requirements-frozen.txt
+ -r{toxinidir}/test-requirements.txt
+
+[testenv:freeze-bionic]
+recreate = True
+allowlist_externals=
+ rm
+ sh
+deps=
+ -r{toxinidir}/requirements-direct.txt
+commands=
+ rm -f {toxinidir}/requirements-frozen-bionic.txt
+ sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen-bionic.txt"
+
[testenv:freeze]
recreate = True
@@ -21,9 +34,25 @@ commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
+[testenv:py36]
+skipsdist=True
+deps =
+ -r{toxinidir}/requirements-frozen-bionic.txt
+ -r{toxinidir}/test-requirements.txt
+commands =
+ pytest \
+ {posargs} \
+ -vv
[testenv:py38]
skipsdist=True
+commands =
+ pytest \
+ {posargs} \
+ -vv
+
+[testenv:cover]
+skipsdist=True
commands =
pytest \
{posargs} \
@@ -31,9 +60,12 @@ commands =
--cov-report=term-missing:skip-covered \
--cov-config=.coveragerc \
--cov=shipyard_client \
- --cov-report=html \
+ --cov-report html:cover \
+ --cov-report xml:cover/coverage.xml \
+ --cov-report term \
-vv
+
[testenv:pep8]
deps=
flake8>=3.3.0
diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml
index 5add4b82..c4d7c3bd 100644
--- a/tools/gate/playbooks/airskiff-deploy.yaml
+++ b/tools/gate/playbooks/airskiff-deploy.yaml
@@ -45,13 +45,15 @@
- name: Build Shipyard and Airflow with submitted changes
shell: |
- make
- docker pull quay.io/airshipit/shipyard:latest-ubuntu_bionic
- docker rm quay.io/airshipit/shipyard:latest-ubuntu_bionic
- docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal quay.io/airshipit/shipyard:latest-ubuntu_bionic
- docker pull quay.io/airshipit/airflow:latest-ubuntu_bionic
- docker rm quay.io/airshipit/airflow:latest-ubuntu_bionic
- docker tag quay.io/airshipit/airflow:latest-ubuntu_focal quay.io/airshipit/airflow:latest-ubuntu_bionic
+ set -ex
+ export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
+ make images
+ if test "${DISTRO}" = 'ubuntu_bionic'
+ then
+ # this trick is needed to use bionic image instead of focal in airskiff deployment test
+ docker tag quay.io/airshipit/airflow:latest-ubuntu_bionic quay.io/airshipit/airflow:latest-ubuntu_focal
+ docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic quay.io/airshipit/shipyard:latest-ubuntu_focal
+ fi
args:
chdir: "{{ zuul.project.src_dir }}"
become: yes
diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml
new file mode 100644
index 00000000..32ccc3b8
--- /dev/null
+++ b/tools/gate/playbooks/docker-image-build.yaml
@@ -0,0 +1,101 @@
+- hosts: all
+ roles:
+ - bindep
+ - ensure-docker
+ - ensure-python
+ - ensure-pip
+ - ensure-tox
+
+ tasks:
+ - include_vars: vars.yaml
+
+ - name: Debug tag generation inputs
+ block:
+ - debug:
+ var: publish
+ - debug:
+ var: tags
+ - debug:
+ var: distro
+ - debug:
+ var: zuul
+ - debug:
+ msg: "{{ tags | to_json }}"
+
+ - name: Determine tags
+ shell: echo '{{ tags | to_json }}' | python3 {{ zuul.project.src_dir }}/tools/image_tags.py
+ environment:
+ BRANCH: "{{ zuul.branch | default('') }}"
+ CHANGE: "{{ zuul.change | default('') }}"
+ COMMIT: "{{ zuul.newrev | default('') }}"
+ PATCHSET: "{{ zuul.patchset | default('') }}"
+ register: image_tags
+
+ - name: Debug computed tags
+ debug:
+ var: image_tags
+
+ - name: Install Docker python module for ansible docker login
+ block:
+ - pip:
+ name: docker
+ version: 4.4.4
+ executable: pip3
+ become: True
+
+ - name: Make images - verbosive
+ when: not publish
+ shell: |
+ make run_images
+ args:
+ chdir: "{{ zuul.project.src_dir }}"
+ executable: /bin/bash
+ become: True
+
+
+ - name: Make images
+ when: not publish
+ block:
+ - make:
+ chdir: "{{ zuul.project.src_dir }}"
+ target: images
+ params:
+ DISTRO: "{{ distro }}"
+ IMAGE_TAG: "{{ item }}"
+ with_items: "{{ image_tags.stdout_lines }}"
+
+ - shell: "docker images"
+ register: docker_images
+
+ - debug:
+ var: docker_images
+
+ become: True
+
+ - name: Publish images
+ block:
+ - docker_login:
+ username: "{{ airship_shipyard_quay_creds.username }}"
+ password: "{{ airship_shipyard_quay_creds.password }}"
+ registry_url: "https://quay.io/api/v1/"
+
+ - make:
+ chdir: "{{ zuul.project.src_dir }}"
+ target: images
+ params:
+ DOCKER_REGISTRY: "quay.io"
+ IMAGE_PREFIX: "airshipit"
+ DISTRO: "{{ distro }}"
+ IMAGE_TAG: "{{ item }}"
+ COMMIT: "{{ zuul.newrev | default('') }}"
+ PUSH_IMAGE: "true"
+ with_items: "{{ image_tags.stdout_lines }}"
+
+ - shell: "docker images"
+ register: docker_images
+
+ - debug:
+ var: docker_images
+
+ when: publish
+ become: True
\ No newline at end of file
diff --git a/tools/gate/roles/build-images/tasks/airship-shipyard.yaml b/tools/gate/roles/build-images/tasks/airship-shipyard.yaml
index 63ec9353..c6f9cadd 100644
--- a/tools/gate/roles/build-images/tasks/airship-shipyard.yaml
+++ b/tools/gate/roles/build-images/tasks/airship-shipyard.yaml
@@ -47,6 +47,17 @@
become: True
+- name: Make and run images - verbosive
+ when: not publish
+ shell: |
+ make run_images
+ args:
+ chdir: "{{ zuul.project.src_dir }}"
+ executable: /bin/bash
+ become: True
+
+
+
- name: Make images
when: not publish
block:
diff --git a/tox.ini b/tox.ini
index 50fc22f1..4a9942da 100644
--- a/tox.ini
+++ b/tox.ini
@@ -49,6 +49,27 @@ commands=
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample"
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample"
+[testenv:py36]
+deps =
+ tox<=4.0.0
+allowlist_externals=
+ tox
+commands=
+ tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py36
+ tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py36
+
+[testenv:cover]
+deps =
+ tox<=4.0.0
+allowlist_externals=
+ tox
+ sh
+commands=
+ tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e cover
+ tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e cover
+ sh -c 'mv ./src/bin/shipyard_airflow/cover ./'
+ sh -c 'mv ./src/bin/shipyard_client/cover ./cover/'
+
[testenv:pep8]
allowlist_externals=
tox