airship/shipyard
Code Issues Proposed changes

Airflow 2.10.2 + ubuntu_jammy

Browse Source 
Change-Id: I88d91ffc9fedb51a61cb34a77e6fd7b9ea187862
This commit is contained in:
Sergiy Markin 2024-09-04 23:19:27 +00:00
parent c8be88cfa2
commit e917c9dca7
23 changed files with 786 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
.zuul.yaml
View File

@ -13,30 +13,34 @@
- project: - project:
templates: templates:
- docs-on-readthedocs - docs-on-readthedocs
- openstack-python38-jobs
vars: vars:
rtd_webhook_id: '38576' rtd_webhook_id: '38576'
rtd_project_name: 'airship-shipyard' rtd_project_name: 'airship-shipyard'
ensure_global_symlinks: true ensure_global_symlinks: true
check: check:
jobs: jobs:
- airship-shipyard-openstack-tox-pep8-focal - openstack-tox-py310
- airship-shipyard-openstack-tox-cover-focal - airship-shipyard-openstack-tox-pep8-jammy
- airship-shipyard-openstack-tox-cover-jammy
- airship-shipyard-chart-build-gate - airship-shipyard-chart-build-gate
- airship-shipyard-chart-build-latest-htk - airship-shipyard-chart-build-latest-htk
- airship-airflow-dependency-vulnerability-check
- airship-shipyard-dependency-vulnerability-check
- airship-shipyard-whitespace-lint-gate - airship-shipyard-whitespace-lint-gate
- airship-shipyard-airskiff-deployment-focal - airship-shipyard-image-gate-ubuntu_jammy
- shipyard-airskiff-deployment-focal-kubeadm - shipyard-airskiff-deployment-jammy-kubeadm
gate: gate:
jobs: jobs:
- airship-shipyard-openstack-tox-pep8-focal - openstack-tox-py310
- airship-shipyard-openstack-tox-cover-focal - airship-shipyard-openstack-tox-pep8-jammy
- airship-shipyard-openstack-tox-cover-jammy
- airship-shipyard-chart-build-gate - airship-shipyard-chart-build-gate
- airship-shipyard-whitespace-lint-gate - airship-shipyard-whitespace-lint-gate
- airship-shipyard-airskiff-deployment-focal - airship-shipyard-image-gate-ubuntu_jammy
- shipyard-airskiff-deployment-jammy-kubeadm
post: post:
jobs: jobs:
- airship-shipyard-docker-build-publish-ubuntu_focal - airship-shipyard-docker-build-publish-ubuntu_jammy
- shipyard-upload-git-mirror - shipyard-upload-git-mirror
- nodeset: - nodeset:
@ -46,32 +50,52 @@
label: ubuntu-bionic label: ubuntu-bionic
- nodeset: - nodeset:
name: airship-shipyard-single-node-focal name: airship-shipyard-single-node-jammy
nodes: nodes:
- name: primary - name: primary
label: ubuntu-focal label: ubuntu-jammy
- job: - job:
name: airship-shipyard-openstack-tox-pep8-focal name: airship-airflow-dependency-vulnerability-check
parent: tox-py310
voting: false
timeout: 600
nodeset: ubuntu-jammy
vars:
tox_envlist: airflow_safety
bindep_profile: test py310
- job:
name: airship-shipyard-dependency-vulnerability-check
parent: tox-py310
voting: false
timeout: 600
nodeset: ubuntu-jammy
vars:
tox_envlist: shipyard_safety
bindep_profile: test py310
- job:
name: airship-shipyard-openstack-tox-pep8-jammy
parent: openstack-tox-pep8 parent: openstack-tox-pep8
description: Runs pep8 job on focal description: Runs pep8 job on jammy
nodeset: airship-shipyard-single-node-focal nodeset: airship-shipyard-single-node-jammy
- job: - job:
name: airship-shipyard-openstack-tox-cover-focal name: airship-shipyard-openstack-tox-cover-jammy
parent: openstack-tox-cover parent: openstack-tox-cover
description: Runs cover job on focal description: Runs cover job on jammy
nodeset: airship-shipyard-single-node-focal nodeset: airship-shipyard-single-node-jammy
- job: - job:
name: airship-shipyard-chart-build-gate name: airship-shipyard-chart-build-gate
description: Build charts using pinned Helm toolkit. description: Build charts using pinned Helm toolkit.
timeout: 900 timeout: 900
run: tools/gate/playbooks/build-charts.yaml run: tools/gate/playbooks/build-charts.yaml
nodeset: airship-shipyard-single-node-focal nodeset: airship-shipyard-single-node-jammy
vars: vars:
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
- job: - job:
name: airship-shipyard-chart-build-latest-htk name: airship-shipyard-chart-build-latest-htk
@ -79,7 +103,7 @@
timeout: 900 timeout: 900
voting: false voting: false
run: tools/gate/playbooks/build-charts.yaml run: tools/gate/playbooks/build-charts.yaml
nodeset: airship-shipyard-single-node-focal nodeset: airship-shipyard-single-node-jammy
vars: vars:
HTK_COMMIT: master HTK_COMMIT: master
@ -88,7 +112,7 @@
description: | description: |
Lints all files by checking them for whitespace. Lints all files by checking them for whitespace.
run: tools/gate/playbooks/zuul-linter.yaml run: tools/gate/playbooks/zuul-linter.yaml
nodeset: airship-shipyard-single-node-focal nodeset: airship-shipyard-single-node-jammy
- job: - job:
name: airship-shipyard-image-base name: airship-shipyard-image-base
@ -96,7 +120,7 @@
Base job for running airship-shipyard image related jobs. Base job for running airship-shipyard image related jobs.
timeout: 3600 timeout: 3600
run: tools/gate/playbooks/docker-image-build.yaml run: tools/gate/playbooks/docker-image-build.yaml
nodeset: airship-shipyard-single-node-focal nodeset: airship-shipyard-single-node-jammy
irrelevant-files: &irrelevant-files irrelevant-files: &irrelevant-files
- ^.*\.rst$ - ^.*\.rst$
- ^doc/.*$ - ^doc/.*$
@ -104,39 +128,17 @@
- job: - job:
name: airship-shipyard-image-gate-ubuntu_focal name: airship-shipyard-image-gate-ubuntu_jammy
description: | description: |
Run shipyard-image build for ubuntu_focal. Run shipyard-image build for ubuntu_jammy.
parent: airship-shipyard-image-base parent: airship-shipyard-image-base
vars: vars:
publish: false publish: false
distro: ubuntu_focal distro: ubuntu_jammy
tags: tags:
dynamic: dynamic:
patch_set: true patch_set: true
# - job:
# name: airship-shipyard-airskiff-deployment-focal
# nodeset: airship-shipyard-single-node-focal
# description: |
# Deploy Memcached using Airskiff and submitted Armada changes.
# timeout: 9600
# voting: true
# pre-run:
# - tools/gate/playbooks/airskiff-reduce-site.yaml
# - tools/gate/playbooks/git-config.yaml
# run: tools/gate/playbooks/airskiff-deploy.yaml
# post-run: tools/gate/playbooks/debug-report.yaml
# required-projects:
# - airship/treasuremap
# vars:
# CLONE_SHIPYARD: false
# SHIPYARD_IMAGE_DISTRO: ubuntu_focal
# HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
# HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
# OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
# OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
# irrelevant-files: *irrelevant-files
- job: - job:
name: shipyard-base name: shipyard-base
@ -153,6 +155,7 @@
irrelevant-files: *irrelevant-files irrelevant-files: *irrelevant-files
timeout: 10800 timeout: 10800
pre-run: pre-run:
- tools/gate/playbooks/checkout-treasuremap-ref.yaml
- tools/gate/playbooks/prepare-hosts.yaml - tools/gate/playbooks/prepare-hosts.yaml
- tools/gate/playbooks/mount-volumes.yaml - tools/gate/playbooks/mount-volumes.yaml
- tools/gate/playbooks/deploy-env.yaml - tools/gate/playbooks/deploy-env.yaml
@ -161,6 +164,7 @@
post-run: post-run:
- tools/gate/playbooks/osh-infra-collect-logs.yaml - tools/gate/playbooks/osh-infra-collect-logs.yaml
vars: vars:
treasuremap_ref: refs/changes/43/927643/4
extra_volume: extra_volume:
size: 80G size: 80G
type: Linux type: Linux
@ -177,16 +181,16 @@
loopback_device: /dev/loop100 loopback_device: /dev/loop100
loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img" loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
ceph_osd_data_device: /dev/loop100 ceph_osd_data_device: /dev/loop100
kube_version_repo: "v1.29" kube_version_repo: "v1.30"
# the list of k8s package versions are available here kube_version: "1.30.3-1.1"
# https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Packages
kube_version: "1.29.5-1.1"
calico_setup: true calico_setup: true
calico_version: "v3.27.0" calico_version: "v3.27.4"
cilium_setup: false cilium_setup: false
helm_version: "v3.14.2" cilium_version: "1.16.0"
yq_version: "v4.6.0" flannel_setup: false
crictl_version: "v1.26.1" flannel_version: v0.25.4
helm_version: "v3.15.4"
crictl_version: "v1.30.1"
zuul_osh_relative_path: ../../openstack/openstack-helm zuul_osh_relative_path: ../../openstack/openstack-helm
zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
zuul_treasuremap_relative_path: ../../airship/treasuremap zuul_treasuremap_relative_path: ../../airship/treasuremap
@ -194,19 +198,20 @@
run_helm_tests: "no" run_helm_tests: "no"
- job: - job:
name: shipyard-airskiff-deployment-focal-kubeadm name: shipyard-airskiff-deployment-jammy-kubeadm
nodeset: treasuremap-airskiff-1node-ubuntu_focal nodeset: treasuremap-airskiff-1node-ubuntu_jammy
description: | description: |
Deploy Memcached using Airskiff and submitted Shipyard changes. Deploy Memcached using Airskiff and submitted Shipyard changes.
parent: shipyard-base parent: shipyard-base
vars: vars:
site: airskiff site: airskiff
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.14.2-linux-amd64.tar.gz HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
HTK_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8 HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
OSH_INFRA_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8 OSH_INFRA_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
OSH_COMMIT: 049e679939fbd3b0c659dd0977911b8dc3b5a015 OSH_COMMIT: 540df5cb0dbdaed63c202e2d6f2b7891062f8203
TREASUREMAP_COMMIT: refs/changes/43/927643/4
CLONE_SHIPYARD: false CLONE_SHIPYARD: false
DISTRO: ubuntu_focal DISTRO: ubuntu_jammy
DOCKER_REGISTRY: localhost:5000 DOCKER_REGISTRY: localhost:5000
MAKE_SHIPYARD_IMAGES: true MAKE_SHIPYARD_IMAGES: true
gate_scripts_relative_path: ../../airship/treasuremap gate_scripts_relative_path: ../../airship/treasuremap
@ -227,23 +232,23 @@
irrelevant-files: *irrelevant-files irrelevant-files: *irrelevant-files
- job: - job:
name: airship-shipyard-docker-build-publish-ubuntu_focal name: airship-shipyard-docker-build-publish-ubuntu_jammy
timeout: 3600 timeout: 3600
run: tools/gate/playbooks/run-image.yaml run: tools/gate/playbooks/run-image.yaml
nodeset: airship-shipyard-single-node-focal nodeset: airship-shipyard-single-node-jammy
irrelevant-files: *irrelevant-files irrelevant-files: *irrelevant-files
secrets: secrets:
- airship_shipyard_quay_creds - airship_shipyard_quay_creds
vars: vars:
publish: true publish: true
distro: ubuntu_focal distro: ubuntu_jammy
tags: tags:
dynamic: dynamic:
branch: true branch: true
commit: true commit: true
static: static:
- latest - latest
- airflow_2.8.2 - airflow_2.10.2

11
Makefile
View File

@ -34,9 +34,11 @@ USE_PROXY ?= false
AIRFLOW_SRC ?= AIRFLOW_SRC ?=
AIRFLOW_HOME ?= AIRFLOW_HOME ?=
DISTRO_BASE_IMAGE ?= DISTRO_BASE_IMAGE ?=
DISTRO ?= ubuntu_focal DISTRO ?= ubuntu_jammy
DISTRO_ALIAS ?= ubuntu_focal
IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO} IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO}
IMAGE_ALIAS := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO_ALIAS}
IMAGE_DIR:=images/$(IMAGE_NAME) IMAGE_DIR:=images/$(IMAGE_NAME)
.PHONY: images .PHONY: images
@ -137,6 +139,13 @@ else
$(_AIRFLOW_HOME_ARG) \ $(_AIRFLOW_HOME_ARG) \
--build-arg ctx_base=$(BUILD_CTX) . --build-arg ctx_base=$(BUILD_CTX) .
endif endif
ifneq ($(DISTRO), $(DISTRO_ALIAS))
docker tag $(IMAGE) $(IMAGE_ALIAS)
endif
ifeq ($(DOCKER_REGISTRY), localhost:5000)
docker push $(IMAGE)
docker push $(IMAGE_ALIAS)
endif
ifeq ($(PUSH_IMAGE), true) ifeq ($(PUSH_IMAGE), true)
docker push $(IMAGE) docker push $(IMAGE)
endif endif

4
charts/shipyard/Chart.yaml
View File

@ -16,8 +16,8 @@
apiVersion: v1 apiVersion: v1
description: A Helm chart for Shipyard and Airflow description: A Helm chart for Shipyard and Airflow
name: shipyard name: shipyard
version: 0.2.6 version: 0.2.7
appVersion: 2.8.2 appVersion: 2.10.2
keywords: keywords:
- shipyard - shipyard
- airflow - airflow

186
images/airflow/Dockerfile.ubuntu_jammy Normal file
View File

@ -0,0 +1,186 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Docker image to run Airflow on Kubernetes
ARG FROM=ubuntu:jammy
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
org.opencontainers.image.url='https://airshipit.org' \
org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \
org.opencontainers.image.source='https://opendev.org/airship/shipyard' \
org.opencontainers.image.vendor='The Airship Authors' \
org.opencontainers.image.licenses='Apache-2.0'
# Do not prompt user for choices on installation/configuration of packages
# Set port 8080 for Airflow Web
# Set port 5555 for Airflow Flower
# Set port 8793 for Airflow Worker
ENV container docker
ENV WEB_PORT 8080
ENV FLOWER_PORT 5555
ENV WORKER_PORT 8793
ENV SLUGIFY_USES_TEXT_UNIDECODE yes
ENV PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
# Expose port for applications
EXPOSE $WEB_PORT
EXPOSE $FLOWER_PORT
EXPOSE $WORKER_PORT
# Set ARG for usage during build
ARG AIRFLOW_HOME=/usr/local/airflow
# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
# version of celery incompatibile with the version of kombu needed by other
# Airship components
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.2"
ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin
# Kubectl version
ARG KUBECTL_VERSION=1.31.0
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
RUN set -ex \
&& apt-get update && apt-get upgrade -y \
&& apt-get install -y wget curl \
apt-transport-https ca-certificates gnupg \
&& echo "deb http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
&& curl -o /etc/apt/trusted.gpg.d/postgresql_release_signing_key.asc 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' \
&& apt update \
&& apt -y install \
automake \
build-essential \
ca-certificates \
curl \
git \
g++ \
libkrb5-dev \
libffi-dev \
libssl-dev \
libpq-dev \
libpq5 \
libtool \
libxml2 \
libsasl2-dev \
locales \
netcat \
netbase \
postgresql-client \
postgresql-common \
python3 \
python3-setuptools \
python3-pip \
python3-dev \
python3-dateutil \
make \
--no-install-recommends \
&& python3 -m pip install -U pip \
&& apt-get clean \
&& rm -rf \
/var/lib/apt/lists/* \
/tmp/* \
/var/tmp/* \
/usr/share/man \
/usr/share/doc \
/usr/share/doc-base \
/etc/ssl/private/ssl-cert-snakeoil.key \
/etc/ssl/private/ssl-cert-snakeoil.pem
# Install LibYAML
ENV LD_LIBRARY_PATH=/usr/local/lib
ARG LIBYAML_VERSION=0.2.5
RUN set -ex \
&& git clone https://github.com/yaml/libyaml.git \
&& cd libyaml \
&& git checkout $LIBYAML_VERSION \
&& ./bootstrap \
&& ./configure \
&& make \
&& make install \
&& cd .. \
&& rm -fr libyaml
# Things that change mostly infrequently
RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
&& curl -L -o /usr/local/bin/kubectl \
https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
&& chmod +x /usr/local/bin/kubectl
# Dependency requirements
# Note - removing snakebite (python 2 vs. 3). See:
# https://github.com/puckel/docker-airflow/issues/77
# Install Airflow directly to allow overriding source
# COPY images/airflow/requirements.txt /tmp/
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/requirements.txt
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
&& pip3 install $AIRFLOW_SRC --no-cache-dir
# && (pip3 uninstall -y snakebite || true) \
# && (pip3 uninstall -y psycopg2 || true) \
# && (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true)
# Copy scripts used in the container:
COPY images/airflow/script/*.sh ${AIRFLOW_HOME}/
# Copy configuration (e.g. logging config for Airflow):
COPY images/airflow/config/*.py ${AIRFLOW_HOME}/config/
COPY images/airflow/webserver_config.py ${AIRFLOW_HOME}/
# Change permissions
RUN chown -R airflow: ${AIRFLOW_HOME}
# Setting the version explicitly for PBR
ENV PBR_VERSION 0.1a1
# Shipyard
#
# Shipyard provides core functionality used by the Airflow plugins/operators
# Since Shipyard and Airflow are built together as images, this should prevent
# stale or out-of-date code between these parts.
# Shipyard requirements, source and installation
# COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt
# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
RUN cd /tmp/shipyard \
&& pip3 install $(pwd) --use-pep517
# Note: The value for the dags and plugins directories that are sourced
# from the values.yaml of the Shipyard Helm chart need to align with these
# directories. If they do not, airflow will not find the intended dags and
# plugins.
#
# Note: In the case of building images using the provided Makefile, a test is
# run against the built-in dags provided with Airflow. Since there is no Helm
# chart to reconfigure the airflow.cfg with these directories, these dags and
# plugins are not known to Airflow during the image test.
#
# Copy the plugins and dags that will be used by this Airflow image:
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins \
/usr/local/lib/python3.10/dist-packages/airflow/plugins/
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags \
/usr/local/lib/python3.10/dist-packages/airflow/dags/
# Set work directory
USER airflow
WORKDIR ${AIRFLOW_HOME}
# Execute entrypoint
ENTRYPOINT ["./entrypoint.sh"]

15
images/airflow/script/airflow_start_service.sh
View File

@ -40,13 +40,26 @@ elif [[ $cmd == *scheduler* ]]; then
${python3_path} ${airflow_path} scheduler $2 $3 ${python3_path} ${airflow_path} scheduler $2 $3
done done
elif [[ $cmd == 'quicktest' ]]; then elif [[ $cmd == 'quicktest' ]]; then
${python3_path} ${airflow_path} version
${python3_path} ${airflow_path} db init ${python3_path} ${airflow_path} db init
${python3_path} ${airflow_path} db migrate ${python3_path} ${airflow_path} db migrate
${python3_path} ${airflow_path} dags list ${python3_path} ${airflow_path} dags list
${python3_path} ${airflow_path} webserver -p 8080 & ${python3_path} ${airflow_path} scheduler &
while true; do
is_active=$(airflow dags details example_bash_operator -o plain | grep is_active | awk '{print $2}')
if [ "$is_active" == "True" ]; then
echo "DAG example_bash_operator is active."
break
fi
echo "Waiting for DAG example_bash_operator to become active..."
sleep 10
done
${python3_path} ${airflow_path} dags list
${python3_path} ${airflow_path} dags unpause example_bash_operator
${python3_path} ${airflow_path} tasks test example_bash_operator runme_0 ${python3_path} ${airflow_path} tasks test example_bash_operator runme_0
${python3_path} ${airflow_path} dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 ${python3_path} ${airflow_path} dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
${python3_path} ${airflow_path} tasks run example_bash_operator runme_0 2018-01-01 ${python3_path} ${airflow_path} tasks run example_bash_operator runme_0 2018-01-01
${python3_path} ${airflow_path} tasks states-for-dag-run example_bash_operator backfill__2018-01-02T00:00:00+00:00
${python3_path} ${airflow_path} dags state example_bash_operator 2018-01-01 ${python3_path} ${airflow_path} dags state example_bash_operator 2018-01-01
else else
echo "Invalid Command!" echo "Invalid Command!"

141
images/shipyard/Dockerfile.ubuntu_jammy Normal file
View File

@ -0,0 +1,141 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ARG FROM=ubuntu:jammy
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
org.opencontainers.image.url='https://airshipit.org' \
org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \
org.opencontainers.image.source='https://opendev.org/airship/shipyard' \
org.opencontainers.image.vendor='The Airship Authors' \
org.opencontainers.image.licenses='Apache-2.0'
ENV container docker
ENV PORT 9000
ENV LC_ALL C.UTF-8
ENV LANG C.UTF-8
ENV PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
# Setting the version explicitly for PBR
ENV PBR_VERSION 0.1a1
ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin
# Expose port 9000 for application
EXPOSE $PORT
RUN set -ex && \
apt-get update && apt-get upgrade -y && \
apt-get -y install \
automake \
ca-certificates \
curl \
build-essential \
git \
libtool \
libxml2 \
make \
netbase \
python3-dev \
python3-setuptools \
--no-install-recommends \
&& apt-get autoremove -yqq --purge \
&& apt-get clean \
&& rm -rf \
/var/lib/apt/lists/* \
/tmp/* \
/var/tmp/* \
/usr/share/man \
/usr/share/doc \
/usr/share/doc-base
# Install LibYAML
ENV LD_LIBRARY_PATH=/usr/local/lib
ARG LIBYAML_VERSION=0.2.5
RUN set -ex \
&& git clone https://github.com/yaml/libyaml.git \
&& cd libyaml \
&& git checkout $LIBYAML_VERSION \
&& ./bootstrap \
&& ./configure \
&& make \
&& make install \
&& cd .. \
&& rm -fr libyaml
# Create shipyard user
RUN useradd -ms /bin/bash shipyard \
&& mkdir -p /home/shipyard/shipyard \
&& mkdir -p /home/shipyard/shipyard_client
# Copy entrypoint.sh to /home/shipyard
COPY ${ctx_base}/shipyard_airflow/entrypoint.sh /home/shipyard/entrypoint.sh
# Change permissions and set up directories
RUN chown -R shipyard: /home/shipyard \
&& chmod +x /home/shipyard/entrypoint.sh
# Requirements and Shipyard source
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /home/shipyard/api_requirements.txt
COPY ${ctx_base}/shipyard_client/requirements-frozen.txt /home/shipyard/client_requirements.txt
COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/
COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
# Build
RUN set -ex \
&& buildDeps=' \
build-essential \
git \
libssl-dev \
libffi-dev \
libkrb5-dev \
libpq-dev \
libsasl2-dev \
libssl-dev \
libre2-dev \
libzmq3-dev \
make \
python3-pip \
' \
&& apt-get -qq update \
&& apt-get upgrade -y \
&& apt-get -y install -y $buildDeps --no-install-recommends \
&& python3 -m pip install -U pip \
&& pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \
&& cd /home/shipyard/shipyard_client \
&& pip3 install . --use-pep517 --no-cache-dir \
&& pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \
&& cd /home/shipyard/shipyard \
&& pip3 install . --use-pep517 --no-cache-dir \
&& apt-get purge -y --auto-remove $buildDeps \
&& apt-get autoremove -yqq --purge \
&& apt-get clean \
&& rm -rf \
/var/lib/apt/lists/* \
/tmp/* \
/var/tmp/* \
/usr/share/man \
/usr/share/doc \
/usr/share/doc-base \
&& python3 -m pip install -U pip
# Entrypoint
ENTRYPOINT ["/home/shipyard/entrypoint.sh"]
CMD ["server"]
# Set user to shipyard
USER shipyard

10
src/bin/shipyard_airflow/requirements-direct.txt
View File

@ -1,5 +1,5 @@
alembic alembic
apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.8.2 apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.2
arrow arrow
celery celery
Flask Flask
@ -59,7 +59,7 @@ oslo.versionedobjects==3.1.0
# Airship dependencies # Airship dependencies
git+https://opendev.org/airship/deckhand.git@32e9950db405b5d1eb74da0c4e8df344a8638eff#egg=deckhand git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6#egg=deckhand
git+https://opendev.org/airship/drydock.git@6b317dda0518e3366c9835ee99d9691ffba718eb#egg=drydock_provisioner&subdirectory=python git+https://opendev.org/airship/drydock.git@146f1debbaf85f66e22cf8c546c3f343f0180ec9#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@e85f0ccdeb93475aef2f2f4ce3da9f53a00309cd#egg=armada git+https://opendev.org/airship/armada.git@edf8b5b88b726691f0dae23ec911e8d539c49211#egg=armada
git+https://opendev.org/airship/promenade.git@d21e3434f641e1f1e7400a44e77fccb5b87a2824#egg=promenade git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66#egg=promenade

301
src/bin/shipyard_airflow/requirements-frozen.txt
View File

@ -1,158 +1,164 @@
aiohttp==3.9.3 aiohappyeyeballs==2.4.0
aiohttp==3.10.5
aiosignal==1.3.1 aiosignal==1.3.1
alembic==1.13.1 alembic==1.13.2
amqp==5.2.0 amqp==5.2.0
anyio==4.3.0 anyio==4.4.0
apache-airflow==2.8.2 apache-airflow==2.10.2
apache-airflow-providers-apache-hdfs==4.3.2 apache-airflow-providers-apache-hdfs==4.5.0
apache-airflow-providers-apache-hive==7.0.0 apache-airflow-providers-apache-hive==8.2.0
apache-airflow-providers-celery==3.6.0 apache-airflow-providers-celery==3.8.1
apache-airflow-providers-common-io==1.3.0 apache-airflow-providers-common-compat==1.2.0
apache-airflow-providers-common-sql==1.11.0 apache-airflow-providers-common-io==1.4.0
apache-airflow-providers-ftp==3.7.0 apache-airflow-providers-common-sql==1.16.0
apache-airflow-providers-http==4.9.1 apache-airflow-providers-fab==1.3.0
apache-airflow-providers-imap==3.5.0 apache-airflow-providers-ftp==3.11.0
apache-airflow-providers-jdbc==4.2.2 apache-airflow-providers-http==4.13.0
apache-airflow-providers-postgres==5.10.1 apache-airflow-providers-imap==3.7.0
apache-airflow-providers-sqlite==3.7.1 apache-airflow-providers-jdbc==4.5.0
apispec==6.4.0 apache-airflow-providers-postgres==5.12.0
argcomplete==3.2.2 apache-airflow-providers-smtp==1.8.0
Armada @ git+https://opendev.org/airship/armada.git@6f06dc09606e94bd6e7322a6d71c1ce5fdd33b0b apache-airflow-providers-sqlite==3.9.0
apispec==6.6.1
argcomplete==3.5.0
Armada @ git+https://opendev.org/airship/armada.git@edf8b5b88b726691f0dae23ec911e8d539c49211
arrow==1.3.0 arrow==1.3.0
asgiref==3.7.2 asgiref==3.8.1
async-timeout==4.0.3 async-timeout==4.0.3
attrs==23.2.0 attrs==24.2.0
autopage==0.5.2 autopage==0.5.2
Babel==2.14.0 babel==2.16.0
backports.zoneinfo==0.2.1
barbican==16.0.0 barbican==16.0.0
bcrypt==4.1.2 bcrypt==4.2.0
Beaker==1.12.1 Beaker==1.13.0
billiard==4.2.0 billiard==4.2.0
blinker==1.7.0 blinker==1.8.2
cachelib==0.9.0 cachelib==0.9.0
cachetools==5.3.2 cachetools==5.5.0
castellan==4.4.0 castellan==5.1.1
celery==5.3.6 celery==5.4.0
certifi==2024.2.2 certifi==2024.8.30
cffi==1.16.0 cffi==1.17.1
charset-normalizer==3.3.2 charset-normalizer==3.3.2
click==8.1.7 click==8.1.7
click-didyoumean==0.3.0 click-didyoumean==0.3.1
click-plugins==1.1.1 click-plugins==1.1.1
click-repl==0.3.0 click-repl==0.3.0
clickclick==20.10.2 clickclick==20.10.2
cliff==4.6.0 cliff==4.7.0
cmd2==2.4.3 cmd2==2.4.3
colorama==0.4.6 colorama==0.4.6
colorlog==4.8.0 colorlog==6.8.2
ConfigUpdater==3.2 ConfigUpdater==3.2
connexion==2.14.2 connexion==2.14.2
cron-descriptor==1.4.3 cron-descriptor==1.4.5
croniter==2.0.1 croniter==3.0.3
cryptography==41.0.7 cryptography==42.0.8
debtcollector==3.0.0 debtcollector==3.0.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@fd58230f6e31ede4925bab0325105eb5b05ad1a8 Deckhand @ git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6
decorator==5.1.1 decorator==5.1.1
deepdiff==6.7.1 deepdiff==8.0.1
Deprecated==1.2.14 Deprecated==1.2.14
dill==0.3.1.1 dill==0.3.1.1
dnspython==2.6.1 dnspython==2.6.1
docopt==0.6.2 docopt==0.6.2
docutils==0.20.1 docutils==0.16
dogpile.cache==1.3.2 dogpile.cache==1.3.3
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@816a4bbe366e2c1d58530986df05f99d7d9a7beb#subdirectory=python drydock_provisioner @ git+https://opendev.org/airship/drydock.git@146f1debbaf85f66e22cf8c546c3f343f0180ec9#subdirectory=python
email-validator==1.3.1 email_validator==2.2.0
eventlet==0.35.2 eventlet==0.37.0
exceptiongroup==1.2.0 exceptiongroup==1.2.2
falcon==3.1.3 falcon==3.1.3
fastavro==1.9.4 fastavro==1.9.7
fasteners==0.19 fasteners==0.19
fixtures==4.1.0 fixtures==4.1.0
Flask==2.2.5 Flask==2.2.5
Flask-AppBuilder==4.3.11 Flask-AppBuilder==4.5.0
Flask-Babel==2.0.0 Flask-Babel==2.0.0
Flask-Caching==2.1.0 Flask-Caching==2.3.0
Flask-JWT-Extended==4.6.0 Flask-JWT-Extended==4.6.0
Flask-Limiter==3.5.1 Flask-Limiter==3.8.0
Flask-Login==0.6.3 Flask-Login==0.6.3
Flask-Session==0.5.0 Flask-Session==0.5.0
Flask-SQLAlchemy==2.5.1 Flask-SQLAlchemy==2.5.1
Flask-WTF==1.2.1 Flask-WTF==1.2.1
flower==2.0.1 flower==2.0.1
frozenlist==1.4.1 frozenlist==1.4.1
fsspec==2024.2.0 fsspec==2024.9.0
future==1.0.0 future==1.0.0
futurist==3.0.0 futurist==3.0.0
gitdb==4.0.11 gitdb==4.0.11
GitPython==3.1.42 GitPython==3.1.43
google-auth==2.28.1 google-auth==2.34.0
google-re2==1.1 google-re2==1.1.20240702
googleapis-common-protos==1.62.0 googleapis-common-protos==1.65.0
graphviz==0.20.1 graphviz==0.20.3
greenlet==3.0.3 greenlet==3.1.0
grpcio==1.62.0 grpcio==1.66.1
gssapi==1.8.3 gssapi==1.8.3
gunicorn==21.2.0 gunicorn==23.0.0
h11==0.14.0 h11==0.14.0
hdfs==2.7.3 hdfs==2.7.3
hmsclient==0.1.1 hmsclient==0.1.1
html5lib==0.9999999 html5lib==0.9999999
httpcore==0.16.3 httpcore==1.0.5
httpexceptor==1.4.0 httpexceptor==1.4.0
httpx==0.23.3 httpx==0.27.0
humanize==4.9.0 humanize==4.10.0
idna==3.6 idna==3.10
importlib-metadata==6.11.0 importlib-metadata==6.11.0
importlib-resources==5.13.0 importlib_resources==6.4.5
inflection==0.5.1 inflection==0.5.1
iso8601==2.1.0 iso8601==2.1.0
itsdangerous==2.1.2 itsdangerous==2.2.0
JayDeBeApi==1.2.3 JayDeBeApi==1.2.3
Jinja2==3.1.3 Jinja2==3.1.4
jmespath==0.10.0
JPype1==1.5.0 JPype1==1.5.0
jsonpath-ng==1.6.1 jsonpath-ng==1.6.1
jsonpickle==3.0.3 jsonpickle==3.3.0
jsonschema==4.21.1 jsonschema==4.23.0
jsonschema-specifications==2023.12.1 jsonschema-specifications==2023.12.1
keystoneauth1==5.1.2 keystoneauth1==5.1.2
keystonemiddleware==10.2.0 keystonemiddleware==10.2.0
kombu==5.3.5 kombu==5.4.1
krb5==0.5.1 krb5==0.6.0
kubernetes==29.0.0 kubernetes==30.1.0
lazy-object-proxy==1.10.0 lazy-object-proxy==1.10.0
ldap3==2.9.1 ldap3==2.9.1
limits==3.9.0 limits==3.13.0
linkify-it-py==2.0.3 linkify-it-py==2.0.3
lockfile==0.12.2 lockfile==0.12.2
logutils==0.3.5 logutils==0.3.5
Mako==1.3.2 Mako==1.3.5
markdown-it-py==3.0.0 markdown-it-py==3.0.0
MarkupSafe==2.1.5 MarkupSafe==2.1.5
marshmallow==3.20.2 marshmallow==3.22.0
marshmallow-oneofschema==3.1.1 marshmallow-oneofschema==3.1.1
marshmallow-sqlalchemy==0.26.1 marshmallow-sqlalchemy==0.28.2
mdit-py-plugins==0.4.0 mdit-py-plugins==0.4.2
mdurl==0.1.2 mdurl==0.1.2
microversion-parse==1.0.1 methodtools==0.4.7
microversion-parse==2.0.0
mock==5.1.0 mock==5.1.0
more-itertools==10.2.0 more-itertools==10.5.0
msgpack==1.0.7 msgpack==1.1.0
multidict==6.0.5 multidict==6.1.0
netaddr==1.2.1 netaddr==1.3.0
netifaces==0.11.0 netifaces==0.11.0
networkx==3.1 networkx==3.3
numpy==1.24.4 numpy==1.26.4
oauthlib==3.2.2 oauthlib==3.2.2
opentelemetry-api==1.23.0 opentelemetry-api==1.27.0
opentelemetry-exporter-otlp==1.23.0 opentelemetry-exporter-otlp==1.27.0
opentelemetry-exporter-otlp-proto-common==1.23.0 opentelemetry-exporter-otlp-proto-common==1.27.0
opentelemetry-exporter-otlp-proto-grpc==1.23.0 opentelemetry-exporter-otlp-proto-grpc==1.27.0
opentelemetry-exporter-otlp-proto-http==1.23.0 opentelemetry-exporter-otlp-proto-http==1.27.0
opentelemetry-proto==1.23.0 opentelemetry-proto==1.27.0
opentelemetry-sdk==1.23.0 opentelemetry-sdk==1.27.0
opentelemetry-semantic-conventions==0.44b0 opentelemetry-semantic-conventions==0.48b0
ordered-set==4.1.0 ordered-set==4.1.0
orderly-set==5.2.2
os-service-types==1.7.0 os-service-types==1.7.0
oslo.cache==3.3.1 oslo.cache==3.3.1
oslo.concurrency==5.1.1 oslo.concurrency==5.1.1
@ -170,115 +176,116 @@ oslo.service==3.1.1
oslo.upgradecheck==2.1.1 oslo.upgradecheck==2.1.1
oslo.utils==6.1.0 oslo.utils==6.1.0
oslo.versionedobjects==3.1.0 oslo.versionedobjects==3.1.0
packaging==23.2 packaging==24.1
pandas==2.0.3 pandas==2.1.4
Paste==3.7.1 Paste==3.10.1
PasteDeploy==3.1.0 PasteDeploy==3.1.0
PasteScript==3.4.0 PasteScript==3.6.0
pathspec==0.12.1 pathspec==0.12.1
pbr==6.0.0 pbr==6.1.0
pecan==1.5.1 pecan==1.5.1
pendulum==3.0.0 pendulum==3.0.0
pip==23.2.1 pip==24.1
pkgutil_resolve_name==1.3.10 pluggy==1.5.0
pluggy==1.4.0
ply==3.11 ply==3.11
prettytable==3.10.0 prettytable==3.11.0
prison==0.2.1 prison==0.2.1
promenade @ git+https://opendev.org/airship/promenade.git@14375e658bc06328042b1dc934eefcebf60210fd promenade @ git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66
prometheus_client==0.20.0 prometheus_client==0.20.0
prompt-toolkit==3.0.43 prompt_toolkit==3.0.47
protobuf==4.25.3 protobuf==4.25.4
psutil==5.9.8 psutil==6.0.0
psycopg2-binary==2.9.9 psycopg2-binary==2.9.9
pure-sasl==0.6.2 pure-sasl==0.6.2
py==1.11.0 py==1.11.0
pyarrow==14.0.2 pyarrow==16.1.0
pyasn1==0.5.1 pyasn1==0.6.1
pyasn1-modules==0.3.0 pyasn1_modules==0.4.0
pycadf==3.1.1 pycadf==3.1.1
pycparser==2.21 pycparser==2.22
Pygments==2.17.2 Pygments==2.18.0
PyHive==0.7.0 PyHive==0.7.0
PyJWT==2.8.0 PyJWT==2.9.0
pylibyaml==0.1.0 pylibyaml==0.1.0
pyOpenSSL==24.0.0 pyOpenSSL==24.2.1
pyparsing==3.1.1 pyparsing==3.1.4
pyperclip==1.8.2 pyperclip==1.9.0
pyspnego==0.10.2 pyspnego==0.11.1
python-barbicanclient==5.5.0 python-barbicanclient==5.5.0
python-daemon==3.0.1 python-daemon==3.0.1
python-dateutil==2.8.2 python-dateutil==2.9.0.post0
python-keystoneclient==5.1.0 python-keystoneclient==5.1.0
python-memcached==1.62 python-memcached==1.62
python-mimeparse==1.6.0 python-mimeparse==2.0.0
python-nvd3==0.15.0 python-nvd3==0.16.0
python-slugify==8.0.4 python-slugify==8.0.4
python3-memcached==1.51 python3-memcached==1.51
pytz==2024.1 pytz==2024.2
PyYAML==6.0.1 PyYAML==6.0.2
referencing==0.33.0 redis==5.0.8
regex==2023.12.25 referencing==0.35.1
regex==2024.9.11
repoze.lru==0.7 repoze.lru==0.7
requests==2.31.0 requests==2.32.3
requests-kerberos==0.14.0 requests-kerberos==0.15.0
requests-oauthlib==1.3.1 requests-oauthlib==1.3.1
requests-toolbelt==1.0.0 requests-toolbelt==1.0.0
resolver==0.2.1 resolver==0.2.1
responses==0.25.0 responses==0.25.3
retry==0.9.2 retry==0.9.2
rfc3339-validator==0.1.4 rfc3339-validator==0.1.4
rfc3986==1.5.0 rfc3986==2.0.0
rich==13.7.0 rich==13.8.1
rich-argparse==1.4.0 rich-argparse==1.5.2
Routes==2.5.1 Routes==2.5.1
rpds-py==0.18.0 rpds-py==0.20.0
rsa==4.9 rsa==4.9
selector==0.10.1 selector==0.10.1
setproctitle==1.3.3 setproctitle==1.3.3
setuptools==68.2.2 setuptools==70.1.0
simplejson==3.19.2 simplejson==3.19.3
six==1.16.0 six==1.16.0
smmap==5.0.1 smmap==5.0.1
sniffio==1.3.0 sniffio==1.3.1
SQLAlchemy==1.4.51 SQLAlchemy==1.4.54
SQLAlchemy-JSONField==1.0.2 SQLAlchemy-JSONField==1.0.2
sqlalchemy-migrate==0.13.0 sqlalchemy-migrate==0.13.0
SQLAlchemy-Utils==0.41.1 SQLAlchemy-Utils==0.41.2
sqlparse==0.4.4 sqlparse==0.5.1
statsd==4.0.1 statsd==4.0.1
stevedore==5.2.0 stevedore==5.3.0
tabulate==0.9.0 tabulate==0.9.0
Tempita==0.5.2 Tempita==0.5.2
tenacity==8.2.3 tenacity==8.5.0
termcolor==2.4.0 termcolor==2.4.0
testresources==2.0.1 testresources==2.0.1
testscenarios==0.5.0 testscenarios==0.5.0
testtools==2.7.1 testtools==2.7.2
text-unidecode==1.3 text-unidecode==1.3
thrift==0.16.0 thrift==0.16.0
thrift-sasl==0.4.3 thrift-sasl==0.4.3
tiddlyweb==2.4.3 tiddlyweb==2.4.3
time-machine==2.13.0 time-machine==2.15.0
tornado==6.4 tornado==6.4.1
types-python-dateutil==2.8.19.20240106 types-python-dateutil==2.9.0.20240906
typing_extensions==4.9.0 typing_extensions==4.12.2
tzdata==2024.1 tzdata==2024.1
uc-micro-py==1.0.3 uc-micro-py==1.0.3
ulid==1.1 ulid==1.1
unicodecsv==0.14.1 unicodecsv==0.14.1
universal-pathlib==0.1.4 universal_pathlib==0.2.5
urllib3==1.26.18 urllib3==2.2.2
uWSGI==2.0.24 uWSGI==2.0.27
vine==5.1.0 vine==5.1.0
wcwidth==0.2.13 wcwidth==0.2.13
WebOb==1.8.7 WebOb==1.8.8
websocket-client==1.7.0 websocket-client==1.8.0
Werkzeug==2.2.3 Werkzeug==2.2.3
wheel==0.41.2 wheel==0.43.0
wirerope==0.4.7
wrapt==1.16.0 wrapt==1.16.0
WTForms==3.1.2 WTForms==3.1.2
xattr==0.10.1 xattr==0.10.1
yappi==1.6.0 yappi==1.6.0
yarl==1.9.4 yarl==1.11.1
zipp==3.17.0 zipp==3.20.2

121
src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_status_operator.py
View File

@ -19,20 +19,24 @@ import yaml
import kubernetes import kubernetes
import airflow import airflow
import pendulum
from shipyard_airflow.common.document_validators import \ from shipyard_airflow.common.document_validators import \
document_validation_utils document_validation_utils
from shipyard_airflow.common.document_validators import errors from shipyard_airflow.common.document_validators import errors
from shipyard_airflow.plugins import deckhand_client_factory from shipyard_airflow.plugins import deckhand_client_factory
from shipyard_airflow.plugins import deployment_status_operator from shipyard_airflow.plugins import deployment_status_operator
from shipyard_airflow.plugins import xcom_puller from shipyard_airflow.plugins import xcom_puller
from airflow import DAG
from airflow.models import TaskInstance, DagRun
from airflow.utils.dates import days_ago
from airflow.utils.state import State
from airflow.utils.types import DagRunType
# @mock.patch.object(airflow.models.BaseOperator, '__init__')
@mock.patch.object(airflow.models.BaseOperator, '__init__')
class TestDeploymentStatusOperator(TestCase): class TestDeploymentStatusOperator(TestCase):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(TestDeploymentStatusOperator, self).__init__(*args, **kwargs) super(TestDeploymentStatusOperator, self).__init__(*args, **kwargs)
self.context = {'ti': None}
self.status = {'status': "doc"} self.status = {'status': "doc"}
self.revision_id = "revision_id" self.revision_id = "revision_id"
self.version = {'version': "doc"} self.version = {'version': "doc"}
@ -42,33 +46,51 @@ class TestDeploymentStatusOperator(TestCase):
} }
self.config_map_data = {'release': yaml.safe_dump(full_data)} self.config_map_data = {'release': yaml.safe_dump(full_data)}
@mock.patch.object(xcom_puller.XcomPuller, 'get_concurrency_status', def setUp(self):
return_value=True) # Create a DAG for the test using pendulum
self.dag = DAG(dag_id="example_dag", start_date=pendulum.now('UTC').add(days=-1))
# Create a DagRun to associate with the TaskInstance
self.dag_run = DagRun(
dag_id=self.dag.dag_id,
execution_date=pendulum.now('UTC').add(days=-1), # Use pendulum for execution_date
run_id="manual__" + pendulum.now('UTC').to_iso8601_string(),
run_type=DagRunType.MANUAL,
state=State.RUNNING
)
# Create a TaskInstance with run_id
self.operator = deployment_status_operator.DeploymentStatusOperator(
shipyard_conf='conf', main_dag_name='name', dag=self.dag, task_id="task")
self.task_instance = TaskInstance(task=self.operator, run_id=self.dag_run.run_id)
self.task_instance.dag_run = self.dag_run # Associate TaskInstance with DagRun
self.task_instance.state = State.RUNNING
# Set the context to include TaskInstance
self.context = {'ti': self.task_instance}
@mock.patch.object(deployment_status_operator.XcomPuller, 'get_concurrency_status', return_value=True)
@mock.patch('shipyard_airflow.conf.config.parse_args') @mock.patch('shipyard_airflow.conf.config.parse_args')
@mock.patch.object(deployment_status_operator.DeploymentStatusOperator, @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_get_version_doc')
'_get_version_doc') @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_get_status_and_revision')
@mock.patch.object(deployment_status_operator.DeploymentStatusOperator, @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_store_as_config_map')
'_get_status_and_revision')
@mock.patch.object(deployment_status_operator.DeploymentStatusOperator,
'_store_as_config_map')
def test_execute(self, def test_execute(self,
store_as_config_map, store_as_config_map,
get_status_and_revision, get_status_and_revision,
get_version_doc, get_version_doc,
config_parse_args, config_parse_args,
xcom_puller, get_concurrency_status_mock):
base_operator_init):
# Mock return values for the methods
get_status_and_revision.return_value = (self.status, self.revision_id) get_status_and_revision.return_value = (self.status, self.revision_id)
get_version_doc.return_value = self.version get_version_doc.return_value = self.version
operator = deployment_status_operator.DeploymentStatusOperator( # Execute the operator
shipyard_conf='conf', main_dag_name='name') self.operator.execute(self.context)
operator.execute(self.context)
assert config_parse_args.called assert config_parse_args.called
assert xcom_puller.called assert get_concurrency_status_mock.called
assert get_status_and_revision.called assert get_status_and_revision.called
get_version_doc.assert_called_once_with(self.revision_id) get_version_doc.assert_called_once_with(self.revision_id)
store_as_config_map.assert_called_once_with(self.config_map_data) store_as_config_map.assert_called_once_with(self.config_map_data)
@ -78,12 +100,10 @@ class TestDeploymentStatusOperator(TestCase):
@mock.patch('shipyard_airflow.conf.config.parse_args') @mock.patch('shipyard_airflow.conf.config.parse_args')
def test_execute_concurrency_fail(self, def test_execute_concurrency_fail(self,
config_parse_args, config_parse_args,
xcom_puller, xcom_puller):
base_operator_init):
operator = deployment_status_operator.DeploymentStatusOperator(
shipyard_conf='conf', main_dag_name='name')
try: try:
operator.execute(self.context) self.operator.execute(self.context)
except airflow.AirflowException as err: except airflow.AirflowException as err:
assert str(err) == "Concurrency check did not pass, so the " \ assert str(err) == "Concurrency check did not pass, so the " \
"deployment status will not be updated" "deployment status will not be updated"
@ -104,15 +124,13 @@ class TestDeploymentStatusOperator(TestCase):
create_cfg_map_object, create_cfg_map_object,
create_cfg_map, create_cfg_map,
patch_cfg_map, patch_cfg_map,
load_k8s_config, load_k8s_config):
base_operator_init):
data = 'data' data = 'data'
patch_cfg_map.side_effect = kubernetes.client.rest.ApiException( patch_cfg_map.side_effect = kubernetes.client.rest.ApiException(
status=404) status=404)
operator = deployment_status_operator.DeploymentStatusOperator(
shipyard_conf='conf', main_dag_name='name') self.operator._store_as_config_map(data)
operator._store_as_config_map(data)
patch_cfg_map.assert_called_once_with( patch_cfg_map.assert_called_once_with(
config_parser.return_value, config_parser.return_value,
config_parser.return_value, config_parser.return_value,
@ -144,13 +162,11 @@ class TestDeploymentStatusOperator(TestCase):
create_cfg_map, create_cfg_map,
patch_cfg_map, patch_cfg_map,
load_k8s_config, load_k8s_config,
get_deployment_status, get_deployment_status):
base_operator_init):
data = 'data' data = 'data'
operator = deployment_status_operator.DeploymentStatusOperator(
shipyard_conf='conf', main_dag_name='name') self.operator._store_as_config_map(data)
operator._store_as_config_map(data)
patch_cfg_map.assert_called_once_with( patch_cfg_map.assert_called_once_with(
config_parser.return_value, config_parser.return_value,
config_parser.return_value, config_parser.return_value,
@ -173,16 +189,14 @@ class TestDeploymentStatusOperator(TestCase):
config_parser, config_parser,
create_cfg_map_object, create_cfg_map_object,
patch_cfg_map, patch_cfg_map,
load_k8s_config, load_k8s_config):
base_operator_init):
data = 'data' data = 'data'
patch_cfg_map.side_effect = kubernetes.client.rest.ApiException( patch_cfg_map.side_effect = kubernetes.client.rest.ApiException(
status=409) status=409)
operator = deployment_status_operator.DeploymentStatusOperator(
shipyard_conf='conf', main_dag_name='name')
try: try:
operator._store_as_config_map(data) self.operator._store_as_config_map(data)
except kubernetes.client.rest.ApiException as err: except kubernetes.client.rest.ApiException as err:
assert patch_cfg_map.side_effect == err assert patch_cfg_map.side_effect == err
@ -206,11 +220,9 @@ class TestDeploymentStatusOperator(TestCase):
def test__get_version_doc(self, def test__get_version_doc(self,
config_parser, config_parser,
get_unique_doc, get_unique_doc,
get_client, get_client):
base_operator_init):
operator = deployment_status_operator.DeploymentStatusOperator( result = self.operator._get_version_doc(self.revision_id)
shipyard_conf='conf', main_dag_name='name')
result = operator._get_version_doc(self.revision_id)
assert result == get_unique_doc.return_value assert result == get_unique_doc.return_value
assert get_client.called assert get_client.called
@ -229,13 +241,11 @@ class TestDeploymentStatusOperator(TestCase):
def test__get_version_doc_does_not_exist(self, def test__get_version_doc_does_not_exist(self,
config_parser, config_parser,
get_unique_doc, get_unique_doc,
get_client, get_client):
base_operator_init):
get_unique_doc.side_effect = errors.DocumentNotFoundError() get_unique_doc.side_effect = errors.DocumentNotFoundError()
operator = deployment_status_operator.DeploymentStatusOperator(
shipyard_conf='conf', main_dag_name='name') result = self.operator._get_version_doc(self.revision_id)
result = operator._get_version_doc(self.revision_id)
assert result == {} assert result == {}
assert get_client.called assert get_client.called
@ -247,21 +257,18 @@ class TestDeploymentStatusOperator(TestCase):
@mock.patch('shipyard_airflow.plugins.deployment_status_operator' @mock.patch('shipyard_airflow.plugins.deployment_status_operator'
'.get_deployment_status') '.get_deployment_status')
def test__get_status_and_revision(self, def test__get_status_and_revision(self,
get_deployment_status, get_deployment_status):
base_operator_init):
operator = deployment_status_operator.DeploymentStatusOperator(
shipyard_conf='conf', main_dag_name='name')
action = {'committed_rev_id': self.revision_id} action = {'committed_rev_id': self.revision_id}
operator.xcom_puller = mock.MagicMock() self.operator.xcom_puller = mock.MagicMock()
operator.xcom_puller.get_action_info = mock.MagicMock() self.operator.xcom_puller.get_action_info = mock.MagicMock()
operator.xcom_puller.get_action_info.return_value = action self.operator.xcom_puller.get_action_info.return_value = action
status_and_revision = operator._get_status_and_revision() status_and_revision = self.operator._get_status_and_revision()
get_deployment_status.assert_called_once_with( get_deployment_status.assert_called_once_with(
action, action,
force_completed=operator.force_completed) force_completed=self.operator.force_completed)
assert status_and_revision == (get_deployment_status.return_value, assert status_and_revision == (get_deployment_status.return_value,
self.revision_id) self.revision_id)

30
src/bin/shipyard_airflow/tox.ini
View File

@ -1,6 +1,6 @@
[tox] [tox]
skipsdist=True skipsdist=True
envlist = py38, pep8, bandit envlist = py310, pep8, bandit
[testenv] [testenv]
setenv= setenv=
@ -17,20 +17,30 @@ allowlist_externals=
sh sh
deps= deps=
-r{toxinidir}/requirements-direct.txt -r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
commands= commands=
rm -f {toxinidir}/requirements-frozen.txt rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt" sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
[testenv:py38] [testenv:safety]
basepython = python3
deps =
safety
allowlist_externals=
safety
commands =
safety check -r {toxinidir}/requirements-frozen.txt --full-report
[testenv:py310]
skipsdist=True skipsdist=True
setenv= setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes SLUGIFY_USES_TEXT_UNIDECODE=yes
PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
basepython=python3.8 basepython=python3.10
allowlist_externals= allowlist_externals=
bash bash
airflow airflow
../../../tools/wait_for_dag.sh
commands = commands =
bash -c "rm -rf $HOME/airflow" bash -c "rm -rf $HOME/airflow"
airflow version airflow version
@ -39,9 +49,14 @@ commands =
airflow info airflow info
airflow dags list airflow dags list
airflow dags list-import-errors airflow dags list-import-errors
bash -c "nohup airflow scheduler >/dev/null 2>&1 &"
../../../tools/wait_for_dag.sh
airflow dags list
airflow dags unpause example_bash_operator
airflow tasks test example_bash_operator runme_0 airflow tasks test example_bash_operator runme_0
airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow tasks run example_bash_operator runme_0 2018-01-01 airflow tasks run example_bash_operator runme_0 2018-01-01
airflow tasks states-for-dag-run example_bash_operator backfill__2018-01-01T00:00:00+00:00
airflow dags state example_bash_operator 2018-01-01 airflow dags state example_bash_operator 2018-01-01
pytest {posargs} -vv pytest {posargs} -vv
@ -53,6 +68,7 @@ setenv=
allowlist_externals= allowlist_externals=
bash bash
airflow airflow
../../../tools/wait_for_dag.sh
commands = commands =
bash -c "rm -rf $HOME/airflow" bash -c "rm -rf $HOME/airflow"
airflow version airflow version
@ -61,9 +77,15 @@ commands =
airflow info airflow info
airflow dags list airflow dags list
airflow dags list-import-errors airflow dags list-import-errors
bash -c "nohup airflow scheduler >/dev/null 2>&1 &"
../../../tools/wait_for_dag.sh
airflow dags list
airflow dags unpause example_bash_operator
airflow tasks test example_bash_operator runme_0 airflow tasks test example_bash_operator runme_0
airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow tasks run example_bash_operator runme_0 2018-01-01 airflow tasks run example_bash_operator runme_0 2018-01-01
airflow tasks states-for-dag-run example_bash_operator backfill__2018-01-02T00:00:00+00:00
airflow dags state example_bash_operator 2018-01-01
pytest \ pytest \
{posargs} \ {posargs} \
--cov-branch \ --cov-branch \

42
src/bin/shipyard_client/requirements-frozen.txt
View File

@ -1,43 +1,41 @@
arrow==1.3.0 arrow==1.3.0
certifi==2024.2.2 certifi==2024.8.30
charset-normalizer==3.3.2 charset-normalizer==3.3.2
click==8.1.7 click==8.1.7
click-default-group==1.2.4 click-default-group==1.2.4
debtcollector==3.0.0 debtcollector==3.0.0
Flask==2.2.5 Flask==2.2.5
idna==3.6 idna==3.10
importlib-metadata==6.11.0
iso8601==2.1.0 iso8601==2.1.0
itsdangerous==2.1.2 itsdangerous==2.2.0
Jinja2==3.1.3 Jinja2==3.1.4
keystoneauth1==5.1.2 keystoneauth1==5.1.2
MarkupSafe==2.1.5 MarkupSafe==2.1.5
msgpack==1.0.7 msgpack==1.1.0
netaddr==1.2.1 netaddr==1.3.0
netifaces==0.11.0 netifaces==0.11.0
os-service-types==1.7.0 os-service-types==1.7.0
oslo.config==9.1.1 oslo.config==9.1.1
oslo.i18n==6.0.0 oslo.i18n==6.0.0
oslo.serialization==5.1.1 oslo.serialization==5.1.1
oslo.utils==6.1.0 oslo.utils==6.1.0
packaging==23.2 packaging==24.1
pbr==6.0.0 pbr==6.1.0
pip==23.2.1 pip==24.1
PTable==0.9.2 PTable==0.9.2
pylibyaml==0.1.0 pylibyaml==0.1.0
pyparsing==3.1.1 pyparsing==3.1.4
python-dateutil==2.8.2 python-dateutil==2.9.0.post0
python-keystoneclient==5.1.0 python-keystoneclient==5.1.0
pytz==2024.1 pytz==2024.2
PyYAML==6.0.1 PyYAML==6.0.2
requests==2.31.0 requests==2.32.3
rfc3986==1.5.0 rfc3986==2.0.0
setuptools==68.2.2 setuptools==70.1.0
six==1.16.0 six==1.16.0
stevedore==5.2.0 stevedore==5.3.0
types-python-dateutil==2.8.19.20240106 types-python-dateutil==2.9.0.20240906
urllib3==1.26.18 urllib3==2.2.2
Werkzeug==2.2.3 Werkzeug==2.2.3
wheel==0.41.2 wheel==0.43.0
wrapt==1.16.0 wrapt==1.16.0
zipp==3.17.0

16
src/bin/shipyard_client/tox.ini
View File

@ -1,6 +1,6 @@
[tox] [tox]
skipsdist=True skipsdist=True
envlist = py38, pep8, bandit envlist = py310, pep8, bandit
[testenv] [testenv]
setenv= setenv=
@ -18,12 +18,22 @@ allowlist_externals=
sh sh
deps= deps=
-r{toxinidir}/requirements-direct.txt -r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
commands= commands=
rm -f {toxinidir}/requirements-frozen.txt rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt" sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
[testenv:py38] [testenv:safety]
basepython = python3
deps =
safety
allowlist_externals=
safety
commands =
safety check -r {toxinidir}/requirements-frozen.txt --full-report
[testenv:py310]
skipsdist=True skipsdist=True
commands = commands =
pytest \ pytest \

4
tools/airflow_image_run.sh
View File

@ -32,6 +32,7 @@ fi
if [ "${USE_PROXY}" == "true" ]; then if [ "${USE_PROXY}" == "true" ]; then
TEST_RESP="$(docker run \ TEST_RESP="$(docker run \
-p 8080:8080 \ -p 8080:8080 \
--rm \
--env HTTP_PROXY="${PROXY}" \ --env HTTP_PROXY="${PROXY}" \
--env HTTPS_PROXY="${PROXY}" \ --env HTTPS_PROXY="${PROXY}" \
--env NO_PROXY="${NO_PROXY}" \ --env NO_PROXY="${NO_PROXY}" \
@ -40,12 +41,11 @@ if [ "${USE_PROXY}" == "true" ]; then
else else
TEST_RESP="$(docker run \ TEST_RESP="$(docker run \
-p 8080:8080 \ -p 8080:8080 \
--rm \
--name airflow_test ${IMAGE} \ --name airflow_test ${IMAGE} \
quicktest)" quicktest)"
fi fi
docker stop airflow_test
docker rm airflow_test
if [ ${TEST_RESP:(-7)} == "success" ]; then if [ ${TEST_RESP:(-7)} == "success" ]; then

31
tools/gate/playbooks/checkout-treasuremap-ref.yaml Normal file
View File

@ -0,0 +1,31 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: all
tasks:
- name: Checkout treasuremap ref
shell: |
set -xe;
: "${TREASUREMAP_REF:=v1.9}"
cd ../treasuremap
git fetch https://review.opendev.org/airship/treasuremap ${TREASUREMAP_REF} && git checkout FETCH_HEAD
args:
chdir: "{{ zuul.project.src_dir }}"
environment:
TREASUREMAP_REF: "{{ treasuremap_ref }}"
...

3
tools/gate/playbooks/prepare-hosts.yaml
View File

@ -15,4 +15,7 @@
roles: roles:
- bindep - bindep
- start-zuul-console - start-zuul-console
- role: add-authorized-keys
public_keys:
- public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com
... ...

2
tools/gate/roles/airship-run-script-set/defaults/main.yaml
View File

@ -19,7 +19,7 @@ osh_params:
container_distro_version: focal container_distro_version: focal
# feature_gates: # feature_gates:
site: airskiff site: airskiff
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7 OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7

2
tools/gate/roles/airship-run-script-set/tasks/main.yaml
View File

@ -35,7 +35,7 @@
FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}" FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}" RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
PL_SITE: "{{ site | default('airskiff') }}" PL_SITE: "{{ site | default('airskiff') }}"
HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}" HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz') }}"
HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}" HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}" OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}" OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"

2
tools/gate/roles/airship-run-script/defaults/main.yaml
View File

@ -19,7 +19,7 @@ osh_params:
container_distro_version: focal container_distro_version: focal
# feature_gates: # feature_gates:
site: airskiff site: airskiff
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7 OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7

2
tools/gate/roles/airship-run-script/tasks/main.yaml
View File

@ -32,7 +32,7 @@
FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}" FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}" RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
PL_SITE: "{{ site | default('airskiff') }}" PL_SITE: "{{ site | default('airskiff') }}"
HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}" HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz') }}"
HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}" HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}" OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}" OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"

2
tools/helm_install.sh
View File

@ -17,7 +17,7 @@
set -x set -x
HELM=$1 HELM=$1
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"} HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz"}
function install_helm_binary { function install_helm_binary {

10
tools/wait_for_dag.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
while true; do
is_active=$(airflow dags details example_bash_operator -o plain | grep is_active | awk '{print $2}')
if [ "$is_active" == "True" ]; then
echo "DAG example_bash_operator is active."
break
fi
echo "Waiting for DAG example_bash_operator to become active..."
sleep 10
done

35
tox.ini
View File

@ -16,6 +16,27 @@ passenv=
[testenv:venv] [testenv:venv]
commands = {posargs} commands = {posargs}
[testenv:airflow_safety]
basepython = python3
deps =
safety
allowlist_externals=
safety
tox
commands =
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e safety
[testenv:shipyard_safety]
basepython = python3
deps =
safety
allowlist_externals=
safety
tox
commands =
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e safety
# Please use `make docs` instead # Please use `make docs` instead
[testenv:docs] [testenv:docs]
allowlist_externals= allowlist_externals=
@ -36,22 +57,22 @@ commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini tox -c {toxinidir}/src/bin/shipyard_client/tox.ini
[testenv:py38] [testenv:py310]
deps = ; deps =
tox<=4.0.0 ; tox
allowlist_externals= allowlist_externals=
tox tox
bash bash
commands= commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py38 tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py310
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py38 tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py310
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample" bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample"
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample" bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample"
[testenv:cover] [testenv:cover]
deps = ; deps =
tox<=4.0.0 ; tox
allowlist_externals= allowlist_externals=
tox tox
sh sh