diff --git a/.zuul.yaml b/.zuul.yaml
index b79c05e6..787b9752 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -13,30 +13,34 @@
- project:
templates:
- docs-on-readthedocs
- - openstack-python38-jobs
vars:
rtd_webhook_id: '38576'
rtd_project_name: 'airship-shipyard'
ensure_global_symlinks: true
check:
jobs:
- - airship-shipyard-openstack-tox-pep8-focal
- - airship-shipyard-openstack-tox-cover-focal
+ - openstack-tox-py310
+ - airship-shipyard-openstack-tox-pep8-jammy
+ - airship-shipyard-openstack-tox-cover-jammy
- airship-shipyard-chart-build-gate
- airship-shipyard-chart-build-latest-htk
+ - airship-airflow-dependency-vulnerability-check
+ - airship-shipyard-dependency-vulnerability-check
- airship-shipyard-whitespace-lint-gate
- - airship-shipyard-airskiff-deployment-focal
- - shipyard-airskiff-deployment-focal-kubeadm
+ - airship-shipyard-image-gate-ubuntu_jammy
+ - shipyard-airskiff-deployment-jammy-kubeadm
gate:
jobs:
- - airship-shipyard-openstack-tox-pep8-focal
- - airship-shipyard-openstack-tox-cover-focal
+ - openstack-tox-py310
+ - airship-shipyard-openstack-tox-pep8-jammy
+ - airship-shipyard-openstack-tox-cover-jammy
- airship-shipyard-chart-build-gate
- airship-shipyard-whitespace-lint-gate
- - airship-shipyard-airskiff-deployment-focal
+ - airship-shipyard-image-gate-ubuntu_jammy
+ - shipyard-airskiff-deployment-jammy-kubeadm
post:
jobs:
- - airship-shipyard-docker-build-publish-ubuntu_focal
+ - airship-shipyard-docker-build-publish-ubuntu_jammy
- shipyard-upload-git-mirror
- nodeset:
@@ -46,32 +50,52 @@
label: ubuntu-bionic
- nodeset:
- name: airship-shipyard-single-node-focal
+ name: airship-shipyard-single-node-jammy
nodes:
- name: primary
- label: ubuntu-focal
+ label: ubuntu-jammy
- job:
- name: airship-shipyard-openstack-tox-pep8-focal
+ name: airship-airflow-dependency-vulnerability-check
+ parent: tox-py310
+ voting: false
+ timeout: 600
+ nodeset: ubuntu-jammy
+ vars:
+ tox_envlist: airflow_safety
+ bindep_profile: test py310
+
+- job:
+ name: airship-shipyard-dependency-vulnerability-check
+ parent: tox-py310
+ voting: false
+ timeout: 600
+ nodeset: ubuntu-jammy
+ vars:
+ tox_envlist: shipyard_safety
+ bindep_profile: test py310
+
+- job:
+ name: airship-shipyard-openstack-tox-pep8-jammy
parent: openstack-tox-pep8
- description: Runs pep8 job on focal
- nodeset: airship-shipyard-single-node-focal
+ description: Runs pep8 job on jammy
+ nodeset: airship-shipyard-single-node-jammy
- job:
- name: airship-shipyard-openstack-tox-cover-focal
+ name: airship-shipyard-openstack-tox-cover-jammy
parent: openstack-tox-cover
- description: Runs cover job on focal
- nodeset: airship-shipyard-single-node-focal
+ description: Runs cover job on jammy
+ nodeset: airship-shipyard-single-node-jammy
- job:
name: airship-shipyard-chart-build-gate
description: Build charts using pinned Helm toolkit.
timeout: 900
run: tools/gate/playbooks/build-charts.yaml
- nodeset: airship-shipyard-single-node-focal
+ nodeset: airship-shipyard-single-node-jammy
vars:
- HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+ HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
- job:
name: airship-shipyard-chart-build-latest-htk
@@ -79,7 +103,7 @@
timeout: 900
voting: false
run: tools/gate/playbooks/build-charts.yaml
- nodeset: airship-shipyard-single-node-focal
+ nodeset: airship-shipyard-single-node-jammy
vars:
HTK_COMMIT: master
@@ -88,7 +112,7 @@
description: |
Lints all files by checking them for whitespace.
run: tools/gate/playbooks/zuul-linter.yaml
- nodeset: airship-shipyard-single-node-focal
+ nodeset: airship-shipyard-single-node-jammy
- job:
name: airship-shipyard-image-base
@@ -96,7 +120,7 @@
Base job for running airship-shipyard image related jobs.
timeout: 3600
run: tools/gate/playbooks/docker-image-build.yaml
- nodeset: airship-shipyard-single-node-focal
+ nodeset: airship-shipyard-single-node-jammy
irrelevant-files: &irrelevant-files
- ^.*\.rst$
- ^doc/.*$
@@ -104,39 +128,17 @@
- job:
- name: airship-shipyard-image-gate-ubuntu_focal
+ name: airship-shipyard-image-gate-ubuntu_jammy
description: |
- Run shipyard-image build for ubuntu_focal.
+ Run shipyard-image build for ubuntu_jammy.
parent: airship-shipyard-image-base
vars:
publish: false
- distro: ubuntu_focal
+ distro: ubuntu_jammy
tags:
dynamic:
patch_set: true
-# - job:
-# name: airship-shipyard-airskiff-deployment-focal
-# nodeset: airship-shipyard-single-node-focal
-# description: |
-# Deploy Memcached using Airskiff and submitted Armada changes.
-# timeout: 9600
-# voting: true
-# pre-run:
-# - tools/gate/playbooks/airskiff-reduce-site.yaml
-# - tools/gate/playbooks/git-config.yaml
-# run: tools/gate/playbooks/airskiff-deploy.yaml
-# post-run: tools/gate/playbooks/debug-report.yaml
-# required-projects:
-# - airship/treasuremap
-# vars:
-# CLONE_SHIPYARD: false
-# SHIPYARD_IMAGE_DISTRO: ubuntu_focal
-# HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
-# HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
-# OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
-# OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
-# irrelevant-files: *irrelevant-files
- job:
name: shipyard-base
@@ -153,6 +155,7 @@
irrelevant-files: *irrelevant-files
timeout: 10800
pre-run:
+ - tools/gate/playbooks/checkout-treasuremap-ref.yaml
- tools/gate/playbooks/prepare-hosts.yaml
- tools/gate/playbooks/mount-volumes.yaml
- tools/gate/playbooks/deploy-env.yaml
@@ -161,6 +164,7 @@
post-run:
- tools/gate/playbooks/osh-infra-collect-logs.yaml
vars:
+ treasuremap_ref: refs/changes/43/927643/4
extra_volume:
size: 80G
type: Linux
@@ -177,16 +181,16 @@
loopback_device: /dev/loop100
loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
ceph_osd_data_device: /dev/loop100
- kube_version_repo: "v1.29"
- # the list of k8s package versions are available here
- # https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Packages
- kube_version: "1.29.5-1.1"
+ kube_version_repo: "v1.30"
+ kube_version: "1.30.3-1.1"
calico_setup: true
- calico_version: "v3.27.0"
+ calico_version: "v3.27.4"
cilium_setup: false
- helm_version: "v3.14.2"
- yq_version: "v4.6.0"
- crictl_version: "v1.26.1"
+ cilium_version: "1.16.0"
+ flannel_setup: false
+ flannel_version: v0.25.4
+ helm_version: "v3.15.4"
+ crictl_version: "v1.30.1"
zuul_osh_relative_path: ../../openstack/openstack-helm
zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
zuul_treasuremap_relative_path: ../../airship/treasuremap
@@ -194,19 +198,20 @@
run_helm_tests: "no"
- job:
- name: shipyard-airskiff-deployment-focal-kubeadm
- nodeset: treasuremap-airskiff-1node-ubuntu_focal
+ name: shipyard-airskiff-deployment-jammy-kubeadm
+ nodeset: treasuremap-airskiff-1node-ubuntu_jammy
description: |
Deploy Memcached using Airskiff and submitted Shipyard changes.
parent: shipyard-base
vars:
site: airskiff
- HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.14.2-linux-amd64.tar.gz
- HTK_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
- OSH_INFRA_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
- OSH_COMMIT: 049e679939fbd3b0c659dd0977911b8dc3b5a015
+ HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
+ HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
+ OSH_INFRA_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
+ OSH_COMMIT: 540df5cb0dbdaed63c202e2d6f2b7891062f8203
+ TREASUREMAP_COMMIT: refs/changes/43/927643/4
CLONE_SHIPYARD: false
- DISTRO: ubuntu_focal
+ DISTRO: ubuntu_jammy
DOCKER_REGISTRY: localhost:5000
MAKE_SHIPYARD_IMAGES: true
gate_scripts_relative_path: ../../airship/treasuremap
@@ -227,23 +232,23 @@
irrelevant-files: *irrelevant-files
- job:
- name: airship-shipyard-docker-build-publish-ubuntu_focal
+ name: airship-shipyard-docker-build-publish-ubuntu_jammy
timeout: 3600
run: tools/gate/playbooks/run-image.yaml
- nodeset: airship-shipyard-single-node-focal
+ nodeset: airship-shipyard-single-node-jammy
irrelevant-files: *irrelevant-files
secrets:
- airship_shipyard_quay_creds
vars:
publish: true
- distro: ubuntu_focal
+ distro: ubuntu_jammy
tags:
dynamic:
branch: true
commit: true
static:
- latest
- - airflow_2.8.2
+ - airflow_2.10.2
diff --git a/Makefile b/Makefile
index 375a3fc7..572a83c1 100644
--- a/Makefile
+++ b/Makefile
@@ -34,9 +34,11 @@ USE_PROXY ?= false
AIRFLOW_SRC ?=
AIRFLOW_HOME ?=
DISTRO_BASE_IMAGE ?=
-DISTRO ?= ubuntu_focal
+DISTRO ?= ubuntu_jammy
+DISTRO_ALIAS ?= ubuntu_focal
IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO}
+IMAGE_ALIAS := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO_ALIAS}
IMAGE_DIR:=images/$(IMAGE_NAME)
.PHONY: images
@@ -137,6 +139,13 @@ else
$(_AIRFLOW_HOME_ARG) \
--build-arg ctx_base=$(BUILD_CTX) .
endif
+ifneq ($(DISTRO), $(DISTRO_ALIAS))
+ docker tag $(IMAGE) $(IMAGE_ALIAS)
+endif
+ifeq ($(DOCKER_REGISTRY), localhost:5000)
+ docker push $(IMAGE)
+ docker push $(IMAGE_ALIAS)
+endif
ifeq ($(PUSH_IMAGE), true)
docker push $(IMAGE)
endif
diff --git a/charts/shipyard/Chart.yaml b/charts/shipyard/Chart.yaml
index 13463b3e..13b17f1e 100644
--- a/charts/shipyard/Chart.yaml
+++ b/charts/shipyard/Chart.yaml
@@ -16,8 +16,8 @@
apiVersion: v1
description: A Helm chart for Shipyard and Airflow
name: shipyard
-version: 0.2.6
-appVersion: 2.8.2
+version: 0.2.7
+appVersion: 2.10.2
keywords:
- shipyard
- airflow
diff --git a/images/airflow/Dockerfile.ubuntu_focal b/images/airflow/Dockerfile.ubuntu_focal
index 3dbe9f08..bd978c67 100644
--- a/images/airflow/Dockerfile.ubuntu_focal
+++ b/images/airflow/Dockerfile.ubuntu_focal
@@ -180,4 +180,4 @@ USER airflow
WORKDIR ${AIRFLOW_HOME}
# Execute entrypoint
-ENTRYPOINT ["./entrypoint.sh"]
+ENTRYPOINT ["./entrypoint.sh"]
\ No newline at end of file
diff --git a/images/airflow/Dockerfile.ubuntu_jammy b/images/airflow/Dockerfile.ubuntu_jammy
new file mode 100644
index 00000000..641ffc21
--- /dev/null
+++ b/images/airflow/Dockerfile.ubuntu_jammy
@@ -0,0 +1,186 @@
+# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Docker image to run Airflow on Kubernetes
+ARG FROM=ubuntu:jammy
+FROM ${FROM}
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
+ org.opencontainers.image.url='https://airshipit.org' \
+ org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \
+ org.opencontainers.image.source='https://opendev.org/airship/shipyard' \
+ org.opencontainers.image.vendor='The Airship Authors' \
+ org.opencontainers.image.licenses='Apache-2.0'
+
+# Do not prompt user for choices on installation/configuration of packages
+# Set port 8080 for Airflow Web
+# Set port 5555 for Airflow Flower
+# Set port 8793 for Airflow Worker
+ENV container docker
+ENV WEB_PORT 8080
+ENV FLOWER_PORT 5555
+ENV WORKER_PORT 8793
+ENV SLUGIFY_USES_TEXT_UNIDECODE yes
+ENV PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
+
+# Expose port for applications
+EXPOSE $WEB_PORT
+EXPOSE $FLOWER_PORT
+EXPOSE $WORKER_PORT
+
+# Set ARG for usage during build
+ARG AIRFLOW_HOME=/usr/local/airflow
+# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
+# version of celery incompatibile with the version of kombu needed by other
+# Airship components
+ARG AIRFLOW_SRC="apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.2"
+ARG DEBIAN_FRONTEND=noninteractive
+ARG ctx_base=src/bin
+
+# Kubectl version
+ARG KUBECTL_VERSION=1.31.0
+
+# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
+ENV AIRFLOW_HOME=${AIRFLOW_HOME}
+
+RUN set -ex \
+ && apt-get update && apt-get upgrade -y \
+ && apt-get install -y wget curl \
+ apt-transport-https ca-certificates gnupg \
+ && echo "deb http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+ && curl -o /etc/apt/trusted.gpg.d/postgresql_release_signing_key.asc 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' \
+ && apt update \
+ && apt -y install \
+ automake \
+ build-essential \
+ ca-certificates \
+ curl \
+ git \
+ g++ \
+ libkrb5-dev \
+ libffi-dev \
+ libssl-dev \
+ libpq-dev \
+ libpq5 \
+ libtool \
+ libxml2 \
+ libsasl2-dev \
+ locales \
+ netcat \
+ netbase \
+ postgresql-client \
+ postgresql-common \
+ python3 \
+ python3-setuptools \
+ python3-pip \
+ python3-dev \
+ python3-dateutil \
+ make \
+ --no-install-recommends \
+ && python3 -m pip install -U pip \
+ && apt-get clean \
+ && rm -rf \
+ /var/lib/apt/lists/* \
+ /tmp/* \
+ /var/tmp/* \
+ /usr/share/man \
+ /usr/share/doc \
+ /usr/share/doc-base \
+ /etc/ssl/private/ssl-cert-snakeoil.key \
+ /etc/ssl/private/ssl-cert-snakeoil.pem
+
+# Install LibYAML
+ENV LD_LIBRARY_PATH=/usr/local/lib
+
+ARG LIBYAML_VERSION=0.2.5
+RUN set -ex \
+ && git clone https://github.com/yaml/libyaml.git \
+ && cd libyaml \
+ && git checkout $LIBYAML_VERSION \
+ && ./bootstrap \
+ && ./configure \
+ && make \
+ && make install \
+ && cd .. \
+ && rm -fr libyaml
+
+
+# Things that change mostly infrequently
+RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
+ && curl -L -o /usr/local/bin/kubectl \
+ https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
+ && chmod +x /usr/local/bin/kubectl
+
+# Dependency requirements
+# Note - removing snakebite (python 2 vs. 3). See:
+# https://github.com/puckel/docker-airflow/issues/77
+# Install Airflow directly to allow overriding source
+# COPY images/airflow/requirements.txt /tmp/
+COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
+ && pip3 install $AIRFLOW_SRC --no-cache-dir
+ # && (pip3 uninstall -y snakebite || true) \
+ # && (pip3 uninstall -y psycopg2 || true) \
+ # && (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true)
+
+# Copy scripts used in the container:
+COPY images/airflow/script/*.sh ${AIRFLOW_HOME}/
+
+# Copy configuration (e.g. logging config for Airflow):
+COPY images/airflow/config/*.py ${AIRFLOW_HOME}/config/
+COPY images/airflow/webserver_config.py ${AIRFLOW_HOME}/
+
+# Change permissions
+RUN chown -R airflow: ${AIRFLOW_HOME}
+
+# Setting the version explicitly for PBR
+ENV PBR_VERSION 0.1a1
+
+# Shipyard
+#
+# Shipyard provides core functionality used by the Airflow plugins/operators
+# Since Shipyard and Airflow are built together as images, this should prevent
+# stale or out-of-date code between these parts.
+# Shipyard requirements, source and installation
+# COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt
+# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
+
+COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
+RUN cd /tmp/shipyard \
+ && pip3 install $(pwd) --use-pep517
+
+# Note: The value for the dags and plugins directories that are sourced
+# from the values.yaml of the Shipyard Helm chart need to align with these
+# directories. If they do not, airflow will not find the intended dags and
+# plugins.
+#
+# Note: In the case of building images using the provided Makefile, a test is
+# run against the built-in dags provided with Airflow. Since there is no Helm
+# chart to reconfigure the airflow.cfg with these directories, these dags and
+# plugins are not known to Airflow during the image test.
+#
+# Copy the plugins and dags that will be used by this Airflow image:
+COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/
+COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins \
+ /usr/local/lib/python3.10/dist-packages/airflow/plugins/
+COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/
+COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags \
+ /usr/local/lib/python3.10/dist-packages/airflow/dags/
+
+# Set work directory
+USER airflow
+WORKDIR ${AIRFLOW_HOME}
+
+# Execute entrypoint
+ENTRYPOINT ["./entrypoint.sh"]
diff --git a/images/airflow/script/airflow_start_service.sh b/images/airflow/script/airflow_start_service.sh
index 82dc2865..eaf62b14 100755
--- a/images/airflow/script/airflow_start_service.sh
+++ b/images/airflow/script/airflow_start_service.sh
@@ -40,13 +40,26 @@ elif [[ $cmd == *scheduler* ]]; then
${python3_path} ${airflow_path} scheduler $2 $3
done
elif [[ $cmd == 'quicktest' ]]; then
+ ${python3_path} ${airflow_path} version
${python3_path} ${airflow_path} db init
${python3_path} ${airflow_path} db migrate
${python3_path} ${airflow_path} dags list
- ${python3_path} ${airflow_path} webserver -p 8080 &
+ ${python3_path} ${airflow_path} scheduler &
+ while true; do
+ is_active=$(airflow dags details example_bash_operator -o plain | grep is_active | awk '{print $2}')
+ if [ "$is_active" == "True" ]; then
+ echo "DAG example_bash_operator is active."
+ break
+ fi
+ echo "Waiting for DAG example_bash_operator to become active..."
+ sleep 10
+ done
+ ${python3_path} ${airflow_path} dags list
+ ${python3_path} ${airflow_path} dags unpause example_bash_operator
${python3_path} ${airflow_path} tasks test example_bash_operator runme_0
${python3_path} ${airflow_path} dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
${python3_path} ${airflow_path} tasks run example_bash_operator runme_0 2018-01-01
+ ${python3_path} ${airflow_path} tasks states-for-dag-run example_bash_operator backfill__2018-01-02T00:00:00+00:00
${python3_path} ${airflow_path} dags state example_bash_operator 2018-01-01
else
echo "Invalid Command!"
diff --git a/images/shipyard/Dockerfile.ubuntu_jammy b/images/shipyard/Dockerfile.ubuntu_jammy
new file mode 100644
index 00000000..99d0e524
--- /dev/null
+++ b/images/shipyard/Dockerfile.ubuntu_jammy
@@ -0,0 +1,141 @@
+# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG FROM=ubuntu:jammy
+FROM ${FROM}
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
+ org.opencontainers.image.url='https://airshipit.org' \
+ org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \
+ org.opencontainers.image.source='https://opendev.org/airship/shipyard' \
+ org.opencontainers.image.vendor='The Airship Authors' \
+ org.opencontainers.image.licenses='Apache-2.0'
+
+ENV container docker
+ENV PORT 9000
+ENV LC_ALL C.UTF-8
+ENV LANG C.UTF-8
+
+ENV PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
+
+# Setting the version explicitly for PBR
+ENV PBR_VERSION 0.1a1
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG ctx_base=src/bin
+
+# Expose port 9000 for application
+EXPOSE $PORT
+
+RUN set -ex && \
+ apt-get update && apt-get upgrade -y && \
+ apt-get -y install \
+ automake \
+ ca-certificates \
+ curl \
+ build-essential \
+ git \
+ libtool \
+ libxml2 \
+ make \
+ netbase \
+ python3-dev \
+ python3-setuptools \
+ --no-install-recommends \
+ && apt-get autoremove -yqq --purge \
+ && apt-get clean \
+ && rm -rf \
+ /var/lib/apt/lists/* \
+ /tmp/* \
+ /var/tmp/* \
+ /usr/share/man \
+ /usr/share/doc \
+ /usr/share/doc-base
+
+# Install LibYAML
+ENV LD_LIBRARY_PATH=/usr/local/lib
+
+ARG LIBYAML_VERSION=0.2.5
+RUN set -ex \
+ && git clone https://github.com/yaml/libyaml.git \
+ && cd libyaml \
+ && git checkout $LIBYAML_VERSION \
+ && ./bootstrap \
+ && ./configure \
+ && make \
+ && make install \
+ && cd .. \
+ && rm -fr libyaml
+
+# Create shipyard user
+RUN useradd -ms /bin/bash shipyard \
+ && mkdir -p /home/shipyard/shipyard \
+ && mkdir -p /home/shipyard/shipyard_client
+
+# Copy entrypoint.sh to /home/shipyard
+COPY ${ctx_base}/shipyard_airflow/entrypoint.sh /home/shipyard/entrypoint.sh
+# Change permissions and set up directories
+RUN chown -R shipyard: /home/shipyard \
+ && chmod +x /home/shipyard/entrypoint.sh
+
+# Requirements and Shipyard source
+COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /home/shipyard/api_requirements.txt
+COPY ${ctx_base}/shipyard_client/requirements-frozen.txt /home/shipyard/client_requirements.txt
+COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/
+COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
+
+# Build
+ RUN set -ex \
+ && buildDeps=' \
+ build-essential \
+ git \
+ libssl-dev \
+ libffi-dev \
+ libkrb5-dev \
+ libpq-dev \
+ libsasl2-dev \
+ libssl-dev \
+ libre2-dev \
+ libzmq3-dev \
+ make \
+ python3-pip \
+ ' \
+ && apt-get -qq update \
+ && apt-get upgrade -y \
+ && apt-get -y install -y $buildDeps --no-install-recommends \
+ && python3 -m pip install -U pip \
+ && pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \
+ && cd /home/shipyard/shipyard_client \
+ && pip3 install . --use-pep517 --no-cache-dir \
+ && pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \
+ && cd /home/shipyard/shipyard \
+ && pip3 install . --use-pep517 --no-cache-dir \
+ && apt-get purge -y --auto-remove $buildDeps \
+ && apt-get autoremove -yqq --purge \
+ && apt-get clean \
+ && rm -rf \
+ /var/lib/apt/lists/* \
+ /tmp/* \
+ /var/tmp/* \
+ /usr/share/man \
+ /usr/share/doc \
+ /usr/share/doc-base \
+ && python3 -m pip install -U pip
+
+
+# Entrypoint
+ENTRYPOINT ["/home/shipyard/entrypoint.sh"]
+CMD ["server"]
+# Set user to shipyard
+USER shipyard
diff --git a/src/bin/shipyard_airflow/requirements-direct.txt b/src/bin/shipyard_airflow/requirements-direct.txt
index 67bb0d53..da8db90e 100644
--- a/src/bin/shipyard_airflow/requirements-direct.txt
+++ b/src/bin/shipyard_airflow/requirements-direct.txt
@@ -1,5 +1,5 @@
alembic
-apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.8.2
+apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.2
arrow
celery
Flask
@@ -59,7 +59,7 @@ oslo.versionedobjects==3.1.0
# Airship dependencies
-git+https://opendev.org/airship/deckhand.git@32e9950db405b5d1eb74da0c4e8df344a8638eff#egg=deckhand
-git+https://opendev.org/airship/drydock.git@6b317dda0518e3366c9835ee99d9691ffba718eb#egg=drydock_provisioner&subdirectory=python
-git+https://opendev.org/airship/armada.git@e85f0ccdeb93475aef2f2f4ce3da9f53a00309cd#egg=armada
-git+https://opendev.org/airship/promenade.git@d21e3434f641e1f1e7400a44e77fccb5b87a2824#egg=promenade
+git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6#egg=deckhand
+git+https://opendev.org/airship/drydock.git@146f1debbaf85f66e22cf8c546c3f343f0180ec9#egg=drydock_provisioner&subdirectory=python
+git+https://opendev.org/airship/armada.git@edf8b5b88b726691f0dae23ec911e8d539c49211#egg=armada
+git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66#egg=promenade
diff --git a/src/bin/shipyard_airflow/requirements-frozen.txt b/src/bin/shipyard_airflow/requirements-frozen.txt
index b4fc34a5..7b6cb32f 100644
--- a/src/bin/shipyard_airflow/requirements-frozen.txt
+++ b/src/bin/shipyard_airflow/requirements-frozen.txt
@@ -1,158 +1,164 @@
-aiohttp==3.9.3
+aiohappyeyeballs==2.4.0
+aiohttp==3.10.5
aiosignal==1.3.1
-alembic==1.13.1
+alembic==1.13.2
amqp==5.2.0
-anyio==4.3.0
-apache-airflow==2.8.2
-apache-airflow-providers-apache-hdfs==4.3.2
-apache-airflow-providers-apache-hive==7.0.0
-apache-airflow-providers-celery==3.6.0
-apache-airflow-providers-common-io==1.3.0
-apache-airflow-providers-common-sql==1.11.0
-apache-airflow-providers-ftp==3.7.0
-apache-airflow-providers-http==4.9.1
-apache-airflow-providers-imap==3.5.0
-apache-airflow-providers-jdbc==4.2.2
-apache-airflow-providers-postgres==5.10.1
-apache-airflow-providers-sqlite==3.7.1
-apispec==6.4.0
-argcomplete==3.2.2
-Armada @ git+https://opendev.org/airship/armada.git@6f06dc09606e94bd6e7322a6d71c1ce5fdd33b0b
+anyio==4.4.0
+apache-airflow==2.10.2
+apache-airflow-providers-apache-hdfs==4.5.0
+apache-airflow-providers-apache-hive==8.2.0
+apache-airflow-providers-celery==3.8.1
+apache-airflow-providers-common-compat==1.2.0
+apache-airflow-providers-common-io==1.4.0
+apache-airflow-providers-common-sql==1.16.0
+apache-airflow-providers-fab==1.3.0
+apache-airflow-providers-ftp==3.11.0
+apache-airflow-providers-http==4.13.0
+apache-airflow-providers-imap==3.7.0
+apache-airflow-providers-jdbc==4.5.0
+apache-airflow-providers-postgres==5.12.0
+apache-airflow-providers-smtp==1.8.0
+apache-airflow-providers-sqlite==3.9.0
+apispec==6.6.1
+argcomplete==3.5.0
+Armada @ git+https://opendev.org/airship/armada.git@edf8b5b88b726691f0dae23ec911e8d539c49211
arrow==1.3.0
-asgiref==3.7.2
+asgiref==3.8.1
async-timeout==4.0.3
-attrs==23.2.0
+attrs==24.2.0
autopage==0.5.2
-Babel==2.14.0
-backports.zoneinfo==0.2.1
+babel==2.16.0
barbican==16.0.0
-bcrypt==4.1.2
-Beaker==1.12.1
+bcrypt==4.2.0
+Beaker==1.13.0
billiard==4.2.0
-blinker==1.7.0
+blinker==1.8.2
cachelib==0.9.0
-cachetools==5.3.2
-castellan==4.4.0
-celery==5.3.6
-certifi==2024.2.2
-cffi==1.16.0
+cachetools==5.5.0
+castellan==5.1.1
+celery==5.4.0
+certifi==2024.8.30
+cffi==1.17.1
charset-normalizer==3.3.2
click==8.1.7
-click-didyoumean==0.3.0
+click-didyoumean==0.3.1
click-plugins==1.1.1
click-repl==0.3.0
clickclick==20.10.2
-cliff==4.6.0
+cliff==4.7.0
cmd2==2.4.3
colorama==0.4.6
-colorlog==4.8.0
+colorlog==6.8.2
ConfigUpdater==3.2
connexion==2.14.2
-cron-descriptor==1.4.3
-croniter==2.0.1
-cryptography==41.0.7
+cron-descriptor==1.4.5
+croniter==3.0.3
+cryptography==42.0.8
debtcollector==3.0.0
-Deckhand @ git+https://opendev.org/airship/deckhand.git@fd58230f6e31ede4925bab0325105eb5b05ad1a8
+Deckhand @ git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6
decorator==5.1.1
-deepdiff==6.7.1
+deepdiff==8.0.1
Deprecated==1.2.14
dill==0.3.1.1
dnspython==2.6.1
docopt==0.6.2
-docutils==0.20.1
-dogpile.cache==1.3.2
-drydock-provisioner @ git+https://opendev.org/airship/drydock.git@816a4bbe366e2c1d58530986df05f99d7d9a7beb#subdirectory=python
-email-validator==1.3.1
-eventlet==0.35.2
-exceptiongroup==1.2.0
+docutils==0.16
+dogpile.cache==1.3.3
+drydock_provisioner @ git+https://opendev.org/airship/drydock.git@146f1debbaf85f66e22cf8c546c3f343f0180ec9#subdirectory=python
+email_validator==2.2.0
+eventlet==0.37.0
+exceptiongroup==1.2.2
falcon==3.1.3
-fastavro==1.9.4
+fastavro==1.9.7
fasteners==0.19
fixtures==4.1.0
Flask==2.2.5
-Flask-AppBuilder==4.3.11
+Flask-AppBuilder==4.5.0
Flask-Babel==2.0.0
-Flask-Caching==2.1.0
+Flask-Caching==2.3.0
Flask-JWT-Extended==4.6.0
-Flask-Limiter==3.5.1
+Flask-Limiter==3.8.0
Flask-Login==0.6.3
Flask-Session==0.5.0
Flask-SQLAlchemy==2.5.1
Flask-WTF==1.2.1
flower==2.0.1
frozenlist==1.4.1
-fsspec==2024.2.0
+fsspec==2024.9.0
future==1.0.0
futurist==3.0.0
gitdb==4.0.11
-GitPython==3.1.42
-google-auth==2.28.1
-google-re2==1.1
-googleapis-common-protos==1.62.0
-graphviz==0.20.1
-greenlet==3.0.3
-grpcio==1.62.0
+GitPython==3.1.43
+google-auth==2.34.0
+google-re2==1.1.20240702
+googleapis-common-protos==1.65.0
+graphviz==0.20.3
+greenlet==3.1.0
+grpcio==1.66.1
gssapi==1.8.3
-gunicorn==21.2.0
+gunicorn==23.0.0
h11==0.14.0
hdfs==2.7.3
hmsclient==0.1.1
html5lib==0.9999999
-httpcore==0.16.3
+httpcore==1.0.5
httpexceptor==1.4.0
-httpx==0.23.3
-humanize==4.9.0
-idna==3.6
+httpx==0.27.0
+humanize==4.10.0
+idna==3.10
importlib-metadata==6.11.0
-importlib-resources==5.13.0
+importlib_resources==6.4.5
inflection==0.5.1
iso8601==2.1.0
-itsdangerous==2.1.2
+itsdangerous==2.2.0
JayDeBeApi==1.2.3
-Jinja2==3.1.3
+Jinja2==3.1.4
+jmespath==0.10.0
JPype1==1.5.0
jsonpath-ng==1.6.1
-jsonpickle==3.0.3
-jsonschema==4.21.1
+jsonpickle==3.3.0
+jsonschema==4.23.0
jsonschema-specifications==2023.12.1
keystoneauth1==5.1.2
keystonemiddleware==10.2.0
-kombu==5.3.5
-krb5==0.5.1
-kubernetes==29.0.0
+kombu==5.4.1
+krb5==0.6.0
+kubernetes==30.1.0
lazy-object-proxy==1.10.0
ldap3==2.9.1
-limits==3.9.0
+limits==3.13.0
linkify-it-py==2.0.3
lockfile==0.12.2
logutils==0.3.5
-Mako==1.3.2
+Mako==1.3.5
markdown-it-py==3.0.0
MarkupSafe==2.1.5
-marshmallow==3.20.2
+marshmallow==3.22.0
marshmallow-oneofschema==3.1.1
-marshmallow-sqlalchemy==0.26.1
-mdit-py-plugins==0.4.0
+marshmallow-sqlalchemy==0.28.2
+mdit-py-plugins==0.4.2
mdurl==0.1.2
-microversion-parse==1.0.1
+methodtools==0.4.7
+microversion-parse==2.0.0
mock==5.1.0
-more-itertools==10.2.0
-msgpack==1.0.7
-multidict==6.0.5
-netaddr==1.2.1
+more-itertools==10.5.0
+msgpack==1.1.0
+multidict==6.1.0
+netaddr==1.3.0
netifaces==0.11.0
-networkx==3.1
-numpy==1.24.4
+networkx==3.3
+numpy==1.26.4
oauthlib==3.2.2
-opentelemetry-api==1.23.0
-opentelemetry-exporter-otlp==1.23.0
-opentelemetry-exporter-otlp-proto-common==1.23.0
-opentelemetry-exporter-otlp-proto-grpc==1.23.0
-opentelemetry-exporter-otlp-proto-http==1.23.0
-opentelemetry-proto==1.23.0
-opentelemetry-sdk==1.23.0
-opentelemetry-semantic-conventions==0.44b0
+opentelemetry-api==1.27.0
+opentelemetry-exporter-otlp==1.27.0
+opentelemetry-exporter-otlp-proto-common==1.27.0
+opentelemetry-exporter-otlp-proto-grpc==1.27.0
+opentelemetry-exporter-otlp-proto-http==1.27.0
+opentelemetry-proto==1.27.0
+opentelemetry-sdk==1.27.0
+opentelemetry-semantic-conventions==0.48b0
ordered-set==4.1.0
+orderly-set==5.2.2
os-service-types==1.7.0
oslo.cache==3.3.1
oslo.concurrency==5.1.1
@@ -170,115 +176,116 @@ oslo.service==3.1.1
oslo.upgradecheck==2.1.1
oslo.utils==6.1.0
oslo.versionedobjects==3.1.0
-packaging==23.2
-pandas==2.0.3
-Paste==3.7.1
+packaging==24.1
+pandas==2.1.4
+Paste==3.10.1
PasteDeploy==3.1.0
-PasteScript==3.4.0
+PasteScript==3.6.0
pathspec==0.12.1
-pbr==6.0.0
+pbr==6.1.0
pecan==1.5.1
pendulum==3.0.0
-pip==23.2.1
-pkgutil_resolve_name==1.3.10
-pluggy==1.4.0
+pip==24.1
+pluggy==1.5.0
ply==3.11
-prettytable==3.10.0
+prettytable==3.11.0
prison==0.2.1
-promenade @ git+https://opendev.org/airship/promenade.git@14375e658bc06328042b1dc934eefcebf60210fd
+promenade @ git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66
prometheus_client==0.20.0
-prompt-toolkit==3.0.43
-protobuf==4.25.3
-psutil==5.9.8
+prompt_toolkit==3.0.47
+protobuf==4.25.4
+psutil==6.0.0
psycopg2-binary==2.9.9
pure-sasl==0.6.2
py==1.11.0
-pyarrow==14.0.2
-pyasn1==0.5.1
-pyasn1-modules==0.3.0
+pyarrow==16.1.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.0
pycadf==3.1.1
-pycparser==2.21
-Pygments==2.17.2
+pycparser==2.22
+Pygments==2.18.0
PyHive==0.7.0
-PyJWT==2.8.0
+PyJWT==2.9.0
pylibyaml==0.1.0
-pyOpenSSL==24.0.0
-pyparsing==3.1.1
-pyperclip==1.8.2
-pyspnego==0.10.2
+pyOpenSSL==24.2.1
+pyparsing==3.1.4
+pyperclip==1.9.0
+pyspnego==0.11.1
python-barbicanclient==5.5.0
python-daemon==3.0.1
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
python-keystoneclient==5.1.0
python-memcached==1.62
-python-mimeparse==1.6.0
-python-nvd3==0.15.0
+python-mimeparse==2.0.0
+python-nvd3==0.16.0
python-slugify==8.0.4
python3-memcached==1.51
-pytz==2024.1
-PyYAML==6.0.1
-referencing==0.33.0
-regex==2023.12.25
+pytz==2024.2
+PyYAML==6.0.2
+redis==5.0.8
+referencing==0.35.1
+regex==2024.9.11
repoze.lru==0.7
-requests==2.31.0
-requests-kerberos==0.14.0
+requests==2.32.3
+requests-kerberos==0.15.0
requests-oauthlib==1.3.1
requests-toolbelt==1.0.0
resolver==0.2.1
-responses==0.25.0
+responses==0.25.3
retry==0.9.2
rfc3339-validator==0.1.4
-rfc3986==1.5.0
-rich==13.7.0
-rich-argparse==1.4.0
+rfc3986==2.0.0
+rich==13.8.1
+rich-argparse==1.5.2
Routes==2.5.1
-rpds-py==0.18.0
+rpds-py==0.20.0
rsa==4.9
selector==0.10.1
setproctitle==1.3.3
-setuptools==68.2.2
-simplejson==3.19.2
+setuptools==70.1.0
+simplejson==3.19.3
six==1.16.0
smmap==5.0.1
-sniffio==1.3.0
-SQLAlchemy==1.4.51
+sniffio==1.3.1
+SQLAlchemy==1.4.54
SQLAlchemy-JSONField==1.0.2
sqlalchemy-migrate==0.13.0
-SQLAlchemy-Utils==0.41.1
-sqlparse==0.4.4
+SQLAlchemy-Utils==0.41.2
+sqlparse==0.5.1
statsd==4.0.1
-stevedore==5.2.0
+stevedore==5.3.0
tabulate==0.9.0
Tempita==0.5.2
-tenacity==8.2.3
+tenacity==8.5.0
termcolor==2.4.0
testresources==2.0.1
testscenarios==0.5.0
-testtools==2.7.1
+testtools==2.7.2
text-unidecode==1.3
thrift==0.16.0
thrift-sasl==0.4.3
tiddlyweb==2.4.3
-time-machine==2.13.0
-tornado==6.4
-types-python-dateutil==2.8.19.20240106
-typing_extensions==4.9.0
+time-machine==2.15.0
+tornado==6.4.1
+types-python-dateutil==2.9.0.20240906
+typing_extensions==4.12.2
tzdata==2024.1
uc-micro-py==1.0.3
ulid==1.1
unicodecsv==0.14.1
-universal-pathlib==0.1.4
-urllib3==1.26.18
-uWSGI==2.0.24
+universal_pathlib==0.2.5
+urllib3==2.2.2
+uWSGI==2.0.27
vine==5.1.0
wcwidth==0.2.13
-WebOb==1.8.7
-websocket-client==1.7.0
+WebOb==1.8.8
+websocket-client==1.8.0
Werkzeug==2.2.3
-wheel==0.41.2
+wheel==0.43.0
+wirerope==0.4.7
wrapt==1.16.0
WTForms==3.1.2
xattr==0.10.1
yappi==1.6.0
-yarl==1.9.4
-zipp==3.17.0
+yarl==1.11.1
+zipp==3.20.2
diff --git a/src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_status_operator.py b/src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_status_operator.py
index c40cb7a8..83838e65 100644
--- a/src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_status_operator.py
+++ b/src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_status_operator.py
@@ -19,20 +19,24 @@ import yaml
import kubernetes
import airflow
+import pendulum
from shipyard_airflow.common.document_validators import \
document_validation_utils
from shipyard_airflow.common.document_validators import errors
from shipyard_airflow.plugins import deckhand_client_factory
from shipyard_airflow.plugins import deployment_status_operator
from shipyard_airflow.plugins import xcom_puller
+from airflow import DAG
+from airflow.models import TaskInstance, DagRun
+from airflow.utils.dates import days_ago
+from airflow.utils.state import State
+from airflow.utils.types import DagRunType
-
-@mock.patch.object(airflow.models.BaseOperator, '__init__')
+# @mock.patch.object(airflow.models.BaseOperator, '__init__')
class TestDeploymentStatusOperator(TestCase):
def __init__(self, *args, **kwargs):
super(TestDeploymentStatusOperator, self).__init__(*args, **kwargs)
- self.context = {'ti': None}
self.status = {'status': "doc"}
self.revision_id = "revision_id"
self.version = {'version': "doc"}
@@ -42,33 +46,51 @@ class TestDeploymentStatusOperator(TestCase):
}
self.config_map_data = {'release': yaml.safe_dump(full_data)}
- @mock.patch.object(xcom_puller.XcomPuller, 'get_concurrency_status',
- return_value=True)
+ def setUp(self):
+ # Create a DAG for the test using pendulum
+ self.dag = DAG(dag_id="example_dag", start_date=pendulum.now('UTC').add(days=-1))
+
+ # Create a DagRun to associate with the TaskInstance
+ self.dag_run = DagRun(
+ dag_id=self.dag.dag_id,
+ execution_date=pendulum.now('UTC').add(days=-1), # Use pendulum for execution_date
+ run_id="manual__" + pendulum.now('UTC').to_iso8601_string(),
+ run_type=DagRunType.MANUAL,
+ state=State.RUNNING
+ )
+
+ # Create a TaskInstance with run_id
+ self.operator = deployment_status_operator.DeploymentStatusOperator(
+ shipyard_conf='conf', main_dag_name='name', dag=self.dag, task_id="task")
+
+ self.task_instance = TaskInstance(task=self.operator, run_id=self.dag_run.run_id)
+ self.task_instance.dag_run = self.dag_run # Associate TaskInstance with DagRun
+ self.task_instance.state = State.RUNNING
+
+ # Set the context to include TaskInstance
+ self.context = {'ti': self.task_instance}
+
+ @mock.patch.object(deployment_status_operator.XcomPuller, 'get_concurrency_status', return_value=True)
@mock.patch('shipyard_airflow.conf.config.parse_args')
- @mock.patch.object(deployment_status_operator.DeploymentStatusOperator,
- '_get_version_doc')
- @mock.patch.object(deployment_status_operator.DeploymentStatusOperator,
- '_get_status_and_revision')
- @mock.patch.object(deployment_status_operator.DeploymentStatusOperator,
- '_store_as_config_map')
+ @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_get_version_doc')
+ @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_get_status_and_revision')
+ @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_store_as_config_map')
def test_execute(self,
store_as_config_map,
get_status_and_revision,
get_version_doc,
config_parse_args,
- xcom_puller,
- base_operator_init):
+ get_concurrency_status_mock):
+ # Mock return values for the methods
get_status_and_revision.return_value = (self.status, self.revision_id)
get_version_doc.return_value = self.version
- operator = deployment_status_operator.DeploymentStatusOperator(
- shipyard_conf='conf', main_dag_name='name')
-
- operator.execute(self.context)
+ # Execute the operator
+ self.operator.execute(self.context)
assert config_parse_args.called
- assert xcom_puller.called
+ assert get_concurrency_status_mock.called
assert get_status_and_revision.called
get_version_doc.assert_called_once_with(self.revision_id)
store_as_config_map.assert_called_once_with(self.config_map_data)
@@ -78,12 +100,10 @@ class TestDeploymentStatusOperator(TestCase):
@mock.patch('shipyard_airflow.conf.config.parse_args')
def test_execute_concurrency_fail(self,
config_parse_args,
- xcom_puller,
- base_operator_init):
- operator = deployment_status_operator.DeploymentStatusOperator(
- shipyard_conf='conf', main_dag_name='name')
+ xcom_puller):
+
try:
- operator.execute(self.context)
+ self.operator.execute(self.context)
except airflow.AirflowException as err:
assert str(err) == "Concurrency check did not pass, so the " \
"deployment status will not be updated"
@@ -104,15 +124,13 @@ class TestDeploymentStatusOperator(TestCase):
create_cfg_map_object,
create_cfg_map,
patch_cfg_map,
- load_k8s_config,
- base_operator_init):
+ load_k8s_config):
data = 'data'
patch_cfg_map.side_effect = kubernetes.client.rest.ApiException(
status=404)
- operator = deployment_status_operator.DeploymentStatusOperator(
- shipyard_conf='conf', main_dag_name='name')
- operator._store_as_config_map(data)
+
+ self.operator._store_as_config_map(data)
patch_cfg_map.assert_called_once_with(
config_parser.return_value,
config_parser.return_value,
@@ -144,13 +162,11 @@ class TestDeploymentStatusOperator(TestCase):
create_cfg_map,
patch_cfg_map,
load_k8s_config,
- get_deployment_status,
- base_operator_init):
+ get_deployment_status):
data = 'data'
- operator = deployment_status_operator.DeploymentStatusOperator(
- shipyard_conf='conf', main_dag_name='name')
- operator._store_as_config_map(data)
+
+ self.operator._store_as_config_map(data)
patch_cfg_map.assert_called_once_with(
config_parser.return_value,
config_parser.return_value,
@@ -173,16 +189,14 @@ class TestDeploymentStatusOperator(TestCase):
config_parser,
create_cfg_map_object,
patch_cfg_map,
- load_k8s_config,
- base_operator_init):
+ load_k8s_config):
data = 'data'
patch_cfg_map.side_effect = kubernetes.client.rest.ApiException(
status=409)
- operator = deployment_status_operator.DeploymentStatusOperator(
- shipyard_conf='conf', main_dag_name='name')
+
try:
- operator._store_as_config_map(data)
+ self.operator._store_as_config_map(data)
except kubernetes.client.rest.ApiException as err:
assert patch_cfg_map.side_effect == err
@@ -206,11 +220,9 @@ class TestDeploymentStatusOperator(TestCase):
def test__get_version_doc(self,
config_parser,
get_unique_doc,
- get_client,
- base_operator_init):
- operator = deployment_status_operator.DeploymentStatusOperator(
- shipyard_conf='conf', main_dag_name='name')
- result = operator._get_version_doc(self.revision_id)
+ get_client):
+
+ result = self.operator._get_version_doc(self.revision_id)
assert result == get_unique_doc.return_value
assert get_client.called
@@ -229,13 +241,11 @@ class TestDeploymentStatusOperator(TestCase):
def test__get_version_doc_does_not_exist(self,
config_parser,
get_unique_doc,
- get_client,
- base_operator_init):
+ get_client):
get_unique_doc.side_effect = errors.DocumentNotFoundError()
- operator = deployment_status_operator.DeploymentStatusOperator(
- shipyard_conf='conf', main_dag_name='name')
- result = operator._get_version_doc(self.revision_id)
+
+ result = self.operator._get_version_doc(self.revision_id)
assert result == {}
assert get_client.called
@@ -247,21 +257,18 @@ class TestDeploymentStatusOperator(TestCase):
@mock.patch('shipyard_airflow.plugins.deployment_status_operator'
'.get_deployment_status')
def test__get_status_and_revision(self,
- get_deployment_status,
- base_operator_init):
- operator = deployment_status_operator.DeploymentStatusOperator(
- shipyard_conf='conf', main_dag_name='name')
+ get_deployment_status):
action = {'committed_rev_id': self.revision_id}
- operator.xcom_puller = mock.MagicMock()
- operator.xcom_puller.get_action_info = mock.MagicMock()
- operator.xcom_puller.get_action_info.return_value = action
+ self.operator.xcom_puller = mock.MagicMock()
+ self.operator.xcom_puller.get_action_info = mock.MagicMock()
+ self.operator.xcom_puller.get_action_info.return_value = action
- status_and_revision = operator._get_status_and_revision()
+ status_and_revision = self.operator._get_status_and_revision()
get_deployment_status.assert_called_once_with(
action,
- force_completed=operator.force_completed)
+ force_completed=self.operator.force_completed)
assert status_and_revision == (get_deployment_status.return_value,
self.revision_id)
diff --git a/src/bin/shipyard_airflow/tox.ini b/src/bin/shipyard_airflow/tox.ini
index 27819e07..dc81b771 100644
--- a/src/bin/shipyard_airflow/tox.ini
+++ b/src/bin/shipyard_airflow/tox.ini
@@ -1,6 +1,6 @@
[tox]
skipsdist=True
-envlist = py38, pep8, bandit
+envlist = py310, pep8, bandit
[testenv]
setenv=
@@ -17,20 +17,30 @@ allowlist_externals=
sh
deps=
-r{toxinidir}/requirements-direct.txt
- -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt
+ -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
-[testenv:py38]
+[testenv:safety]
+basepython = python3
+deps =
+ safety
+allowlist_externals=
+ safety
+commands =
+ safety check -r {toxinidir}/requirements-frozen.txt --full-report
+
+[testenv:py310]
skipsdist=True
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
-basepython=python3.8
+basepython=python3.10
allowlist_externals=
bash
airflow
+ ../../../tools/wait_for_dag.sh
commands =
bash -c "rm -rf $HOME/airflow"
airflow version
@@ -39,9 +49,14 @@ commands =
airflow info
airflow dags list
airflow dags list-import-errors
+ bash -c "nohup airflow scheduler >/dev/null 2>&1 &"
+ ../../../tools/wait_for_dag.sh
+ airflow dags list
+ airflow dags unpause example_bash_operator
airflow tasks test example_bash_operator runme_0
airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow tasks run example_bash_operator runme_0 2018-01-01
+ airflow tasks states-for-dag-run example_bash_operator backfill__2018-01-01T00:00:00+00:00
airflow dags state example_bash_operator 2018-01-01
pytest {posargs} -vv
@@ -53,6 +68,7 @@ setenv=
allowlist_externals=
bash
airflow
+ ../../../tools/wait_for_dag.sh
commands =
bash -c "rm -rf $HOME/airflow"
airflow version
@@ -61,9 +77,15 @@ commands =
airflow info
airflow dags list
airflow dags list-import-errors
+ bash -c "nohup airflow scheduler >/dev/null 2>&1 &"
+ ../../../tools/wait_for_dag.sh
+ airflow dags list
+ airflow dags unpause example_bash_operator
airflow tasks test example_bash_operator runme_0
airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow tasks run example_bash_operator runme_0 2018-01-01
+ airflow tasks states-for-dag-run example_bash_operator backfill__2018-01-02T00:00:00+00:00
+ airflow dags state example_bash_operator 2018-01-01
pytest \
{posargs} \
--cov-branch \
diff --git a/src/bin/shipyard_client/requirements-frozen.txt b/src/bin/shipyard_client/requirements-frozen.txt
index 8ad50451..c524b9c0 100644
--- a/src/bin/shipyard_client/requirements-frozen.txt
+++ b/src/bin/shipyard_client/requirements-frozen.txt
@@ -1,43 +1,41 @@
arrow==1.3.0
-certifi==2024.2.2
+certifi==2024.8.30
charset-normalizer==3.3.2
click==8.1.7
click-default-group==1.2.4
debtcollector==3.0.0
Flask==2.2.5
-idna==3.6
-importlib-metadata==6.11.0
+idna==3.10
iso8601==2.1.0
-itsdangerous==2.1.2
-Jinja2==3.1.3
+itsdangerous==2.2.0
+Jinja2==3.1.4
keystoneauth1==5.1.2
MarkupSafe==2.1.5
-msgpack==1.0.7
-netaddr==1.2.1
+msgpack==1.1.0
+netaddr==1.3.0
netifaces==0.11.0
os-service-types==1.7.0
oslo.config==9.1.1
oslo.i18n==6.0.0
oslo.serialization==5.1.1
oslo.utils==6.1.0
-packaging==23.2
-pbr==6.0.0
-pip==23.2.1
+packaging==24.1
+pbr==6.1.0
+pip==24.1
PTable==0.9.2
pylibyaml==0.1.0
-pyparsing==3.1.1
-python-dateutil==2.8.2
+pyparsing==3.1.4
+python-dateutil==2.9.0.post0
python-keystoneclient==5.1.0
-pytz==2024.1
-PyYAML==6.0.1
-requests==2.31.0
-rfc3986==1.5.0
-setuptools==68.2.2
+pytz==2024.2
+PyYAML==6.0.2
+requests==2.32.3
+rfc3986==2.0.0
+setuptools==70.1.0
six==1.16.0
-stevedore==5.2.0
-types-python-dateutil==2.8.19.20240106
-urllib3==1.26.18
+stevedore==5.3.0
+types-python-dateutil==2.9.0.20240906
+urllib3==2.2.2
Werkzeug==2.2.3
-wheel==0.41.2
+wheel==0.43.0
wrapt==1.16.0
-zipp==3.17.0
diff --git a/src/bin/shipyard_client/tox.ini b/src/bin/shipyard_client/tox.ini
index c8e8d02f..2d08d4cf 100644
--- a/src/bin/shipyard_client/tox.ini
+++ b/src/bin/shipyard_client/tox.ini
@@ -1,6 +1,6 @@
[tox]
skipsdist=True
-envlist = py38, pep8, bandit
+envlist = py310, pep8, bandit
[testenv]
setenv=
@@ -18,12 +18,22 @@ allowlist_externals=
sh
deps=
-r{toxinidir}/requirements-direct.txt
- -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt
+ -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
-[testenv:py38]
+[testenv:safety]
+basepython = python3
+deps =
+ safety
+allowlist_externals=
+ safety
+commands =
+ safety check -r {toxinidir}/requirements-frozen.txt --full-report
+
+
+[testenv:py310]
skipsdist=True
commands =
pytest \
diff --git a/tools/airflow_image_run.sh b/tools/airflow_image_run.sh
index 6b19b284..9169bef4 100755
--- a/tools/airflow_image_run.sh
+++ b/tools/airflow_image_run.sh
@@ -32,6 +32,7 @@ fi
if [ "${USE_PROXY}" == "true" ]; then
TEST_RESP="$(docker run \
-p 8080:8080 \
+ --rm \
--env HTTP_PROXY="${PROXY}" \
--env HTTPS_PROXY="${PROXY}" \
--env NO_PROXY="${NO_PROXY}" \
@@ -40,12 +41,11 @@ if [ "${USE_PROXY}" == "true" ]; then
else
TEST_RESP="$(docker run \
-p 8080:8080 \
+ --rm \
--name airflow_test ${IMAGE} \
quicktest)"
fi
-docker stop airflow_test
-docker rm airflow_test
if [ ${TEST_RESP:(-7)} == "success" ]; then
diff --git a/tools/gate/playbooks/checkout-treasuremap-ref.yaml b/tools/gate/playbooks/checkout-treasuremap-ref.yaml
new file mode 100644
index 00000000..46a24051
--- /dev/null
+++ b/tools/gate/playbooks/checkout-treasuremap-ref.yaml
@@ -0,0 +1,31 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+
+ tasks:
+ - name: Checkout treasuremap ref
+ shell: |
+ set -xe;
+
+ : "${TREASUREMAP_REF:=v1.9}"
+
+ cd ../treasuremap
+ git fetch https://review.opendev.org/airship/treasuremap ${TREASUREMAP_REF} && git checkout FETCH_HEAD
+
+ args:
+ chdir: "{{ zuul.project.src_dir }}"
+ environment:
+ TREASUREMAP_REF: "{{ treasuremap_ref }}"
+
+...
\ No newline at end of file
diff --git a/tools/gate/playbooks/prepare-hosts.yaml b/tools/gate/playbooks/prepare-hosts.yaml
index e7e3d9c9..76087fde 100644
--- a/tools/gate/playbooks/prepare-hosts.yaml
+++ b/tools/gate/playbooks/prepare-hosts.yaml
@@ -15,4 +15,7 @@
roles:
- bindep
- start-zuul-console
+ - role: add-authorized-keys
+ public_keys:
+ - public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com
...
diff --git a/tools/gate/roles/airship-run-script-set/defaults/main.yaml b/tools/gate/roles/airship-run-script-set/defaults/main.yaml
index 725f4b23..7dc55afd 100644
--- a/tools/gate/roles/airship-run-script-set/defaults/main.yaml
+++ b/tools/gate/roles/airship-run-script-set/defaults/main.yaml
@@ -19,7 +19,7 @@ osh_params:
container_distro_version: focal
# feature_gates:
site: airskiff
-HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
diff --git a/tools/gate/roles/airship-run-script-set/tasks/main.yaml b/tools/gate/roles/airship-run-script-set/tasks/main.yaml
index 4129a2c1..0fe778cd 100644
--- a/tools/gate/roles/airship-run-script-set/tasks/main.yaml
+++ b/tools/gate/roles/airship-run-script-set/tasks/main.yaml
@@ -35,7 +35,7 @@
FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
PL_SITE: "{{ site | default('airskiff') }}"
- HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}"
+ HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz') }}"
HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
diff --git a/tools/gate/roles/airship-run-script/defaults/main.yaml b/tools/gate/roles/airship-run-script/defaults/main.yaml
index 725f4b23..7dc55afd 100644
--- a/tools/gate/roles/airship-run-script/defaults/main.yaml
+++ b/tools/gate/roles/airship-run-script/defaults/main.yaml
@@ -19,7 +19,7 @@ osh_params:
container_distro_version: focal
# feature_gates:
site: airskiff
-HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
diff --git a/tools/gate/roles/airship-run-script/tasks/main.yaml b/tools/gate/roles/airship-run-script/tasks/main.yaml
index 006106b3..0220ef49 100644
--- a/tools/gate/roles/airship-run-script/tasks/main.yaml
+++ b/tools/gate/roles/airship-run-script/tasks/main.yaml
@@ -32,7 +32,7 @@
FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
PL_SITE: "{{ site | default('airskiff') }}"
- HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}"
+ HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz') }}"
HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
diff --git a/tools/helm_install.sh b/tools/helm_install.sh
index 9c0dae59..77171564 100755
--- a/tools/helm_install.sh
+++ b/tools/helm_install.sh
@@ -17,7 +17,7 @@
set -x
HELM=$1
-HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"}
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz"}
function install_helm_binary {
diff --git a/tools/wait_for_dag.sh b/tools/wait_for_dag.sh
new file mode 100755
index 00000000..81f11c2e
--- /dev/null
+++ b/tools/wait_for_dag.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+while true; do
+ is_active=$(airflow dags details example_bash_operator -o plain | grep is_active | awk '{print $2}')
+ if [ "$is_active" == "True" ]; then
+ echo "DAG example_bash_operator is active."
+ break
+ fi
+ echo "Waiting for DAG example_bash_operator to become active..."
+ sleep 10
+done
\ No newline at end of file
diff --git a/tox.ini b/tox.ini
index 7c0bfb50..40e1aaea 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,6 +16,27 @@ passenv=
[testenv:venv]
commands = {posargs}
+[testenv:airflow_safety]
+basepython = python3
+deps =
+ safety
+allowlist_externals=
+ safety
+ tox
+commands =
+ tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e safety
+
+[testenv:shipyard_safety]
+basepython = python3
+deps =
+ safety
+allowlist_externals=
+ safety
+ tox
+commands =
+ tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e safety
+
+
# Please use `make docs` instead
[testenv:docs]
allowlist_externals=
@@ -36,22 +57,22 @@ commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini
-[testenv:py38]
-deps =
- tox<=4.0.0
+[testenv:py310]
+; deps =
+; tox
allowlist_externals=
tox
bash
commands=
- tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py38
- tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py38
+ tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py310
+ tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py310
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample"
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample"
[testenv:cover]
-deps =
- tox<=4.0.0
+; deps =
+; tox
allowlist_externals=
tox
sh