airship/shipyard
Code Issues Proposed changes

Build workflows into Airflow image

Browse Source 
Changes to make the docker image build to include the workflows from
Shipyard, rather than adding them to the container during Helm install
of Shipyard. This also removes the "prod" switch, as it is now always
built the same way, with the workflows in place.

Change-Id: I4acd6195cbec32193e15621e75ccaeb9879455f5
This commit is contained in:
Bryan Strassner 2019-01-17 13:55:40 -06:00
parent 6b75c7119a
commit 9725b0f337
7 changed files with 15 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl
View File

@ -1,23 +0,0 @@
#!/bin/bash
{{/*
Copyright 2017 The Openstack-Helm Authors.
Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
cp -vaR /home/shipyard/shipyard/shipyard_airflow/dags/*.py /tmp/airflow/dags/
cp -vaR /home/shipyard/shipyard/shipyard_airflow/plugins/*.py /tmp/airflow/plugins/

2
charts/shipyard/templates/configmap-airflow-bin.yaml
View File

@ -27,6 +27,4 @@ data:
{{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
airflow-db-sync.sh: |+
{{ tuple "bin/_airflow-db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
airflow-shipyard-init.sh: |+
{{ tuple "bin/_airflow-shipyard-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

44
charts/shipyard/templates/deployment-airflow-scheduler.yaml
View File

@ -53,23 +53,6 @@ spec:
terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }}
initContainers:
{{ tuple $envAll "airflow_server" $mounts_airflow_scheduler_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{- if .Values.prod_environment }}
- name: airflow-shipyard-init
image: {{ .Values.images.tags.shipyard }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /tmp/airflow-shipyard-init.sh
volumeMounts:
- name: airflow-bin
mountPath: /tmp/airflow-shipyard-init.sh
subPath: airflow-shipyard-init.sh
readOnly: true
- name: pod-shipyard-share-airflow-dags
mountPath: /tmp/airflow/dags/
- name: pod-shipyard-share-airflow-plugins
mountPath: /tmp/airflow/plugins/
{{- end }}
containers:
- name: airflow-scheduler
image: {{ .Values.images.tags.airflow }}
@ -88,21 +71,10 @@ spec:
mountPath: {{ .Values.conf.airflow_config_file.path }}
subPath: airflow.cfg
readOnly: true
{{- if .Values.prod_environment }}
- name: shipyard-etc
mountPath: /usr/local/airflow/plugins/shipyard.conf
subPath: shipyard.conf
readOnly: true
- name: pod-shipyard-share-airflow-dags
mountPath: {{ .Values.conf.airflow.core.dags_folder }}
- name: pod-shipyard-share-airflow-plugins
mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
{{ else }}
- name: airflow-dags
mountPath: {{ .Values.conf.airflow.core.dags_folder }}
- name: airflow-plugins
mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
{{- end }}
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
{{ if $mounts_airflow_scheduler.volumeMounts }}{{ toYaml $mounts_airflow_scheduler.volumeMounts | indent 12 }}{{ end }}
@ -111,7 +83,6 @@ spec:
configMap:
name: airflow-etc
defaultMode: 0444
{{- if .Values.prod_environment }}
- name: shipyard-etc
configMap:
name: shipyard-etc
@ -120,22 +91,7 @@ spec:
configMap:
name: airflow-bin
defaultMode: 0555
- name: pod-shipyard-share-airflow-dags
emptyDir: {}
- name: pod-shipyard-share-airflow-plugins
emptyDir: {}
- name: airflow-logs
emptyDir: {}
{{ else }}
- name: airflow-dags
hostPath:
path: {{ .Values.pod.mounts.dag_path }}
- name: airflow-plugins
hostPath:
path: {{ .Values.pod.mounts.plugin_path }}
- name: airflow-logs
hostPath:
path: {{ .Values.pod.mounts.log_path }}
{{- end }}
{{ if $mounts_airflow_scheduler.volumes }}{{ toYaml $mounts_airflow_scheduler.volumes | indent 8 }}{{ end }}
{{- end }}

44
charts/shipyard/templates/deployment-airflow-web.yaml
View File

@ -44,23 +44,6 @@ spec:
terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }}
initContainers:
{{ tuple $envAll "airflow_server" $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{- if .Values.prod_environment }}
- name: airflow-shipyard-init
image: {{ .Values.images.tags.shipyard }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /tmp/airflow-shipyard-init.sh
volumeMounts:
- name: airflow-bin
mountPath: /tmp/airflow-shipyard-init.sh
subPath: airflow-shipyard-init.sh
readOnly: true
- name: pod-shipyard-share-airflow-dags
mountPath: /tmp/airflow/dags/
- name: pod-shipyard-share-airflow-plugins
mountPath: /tmp/airflow/plugins/
{{- end }}
containers:
- name: airflow-web
image: {{ .Values.images.tags.airflow }}
@ -83,21 +66,10 @@ spec:
mountPath: {{ .Values.conf.airflow_config_file.path }}
subPath: airflow.cfg
readOnly: true
{{- if .Values.prod_environment }}
- name: shipyard-etc
mountPath: /usr/local/airflow/plugins/shipyard.conf
subPath: shipyard.conf
readOnly: true
- name: pod-shipyard-share-airflow-dags
mountPath: {{ .Values.conf.airflow.core.dags_folder }}
- name: pod-shipyard-share-airflow-plugins
mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
{{ else }}
- name: airflow-dags
mountPath: {{ .Values.conf.airflow.core.dags_folder }}
- name: airflow-plugins
mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
{{- end }}
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
{{ if $mounts_airflow_web.volumeMounts }}{{ toYaml $mounts_airflow_web.volumeMounts | indent 12 }}{{ end }}
@ -106,7 +78,6 @@ spec:
configMap:
name: airflow-etc
defaultMode: 0444
{{- if .Values.prod_environment }}
- name: shipyard-etc
configMap:
name: shipyard-etc
@ -115,22 +86,7 @@ spec:
configMap:
name: airflow-bin
defaultMode: 0555
- name: pod-shipyard-share-airflow-dags
emptyDir: {}
- name: pod-shipyard-share-airflow-plugins
emptyDir: {}
- name: airflow-logs
emptyDir: {}
{{ else }}
- name: airflow-dags
hostPath:
path: {{ .Values.pod.mounts.dag_path }}
- name: airflow-plugins
hostPath:
path: {{ .Values.pod.mounts.plugin_path }}
- name: airflow-logs
hostPath:
path: {{ .Values.pod.mounts.log_path }}
{{- end }}
{{ if $mounts_airflow_web.volumes }}{{ toYaml $mounts_airflow_web.volumes | indent 8 }}{{ end }}
{{- end }}

69
charts/shipyard/templates/statefulset-airflow-worker.yaml
View File

@ -79,22 +79,6 @@ spec:
{{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }}
initContainers:
{{ tuple $envAll "airflow_server" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{- if .Values.prod_environment }}
- name: airflow-shipyard-init
image: {{ .Values.images.tags.shipyard }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /tmp/airflow-shipyard-init.sh
volumeMounts:
- name: airflow-bin
mountPath: /tmp/airflow-shipyard-init.sh
subPath: airflow-shipyard-init.sh
readOnly: true
- name: pod-shipyard-share-airflow-dags
mountPath: /tmp/airflow/dags/
- name: pod-shipyard-share-airflow-plugins
mountPath: /tmp/airflow/plugins/
- name: worker-perms
image: {{ .Values.images.tags.airflow }}
imagePullPolicy: {{ .Values.images.pull_policy }}
@ -109,7 +93,6 @@ spec:
volumeMounts:
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
{{- end }}
containers:
- name: airflow-scheduler
image: {{ .Values.images.tags.airflow }}
@ -128,21 +111,10 @@ spec:
mountPath: {{ .Values.conf.airflow_config_file.path }}
subPath: airflow.cfg
readOnly: true
{{- if .Values.prod_environment }}
- name: shipyard-etc
mountPath: /usr/local/airflow/plugins/shipyard.conf
subPath: shipyard.conf
readOnly: true
- name: pod-shipyard-share-airflow-dags
mountPath: {{ .Values.conf.airflow.core.dags_folder }}
- name: pod-shipyard-share-airflow-plugins
mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
{{ else }}
- name: airflow-dags
mountPath: {{ .Values.conf.airflow.core.dags_folder }}
- name: airflow-plugins
mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
{{- end }}
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
- name: airflow-worker
@ -166,30 +138,12 @@ spec:
mountPath: {{ .Values.conf.airflow_config_file.path }}
subPath: airflow.cfg
readOnly: true
{{- if .Values.prod_environment }}
- name: shipyard-etc
mountPath: /usr/local/airflow/plugins/shipyard.conf
subPath: shipyard.conf
readOnly: true
- name: pod-shipyard-share-airflow-dags
mountPath: {{ .Values.conf.airflow.core.dags_folder }}
- name: pod-shipyard-share-airflow-plugins
mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
{{ else }}
- name: airflow-dags
mountPath: {{ .Values.conf.airflow.core.dags_folder }}
- name: airflow-plugins
mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
- name: docker
mountPath: /var/run
readOnly: false
- name: pod-var-lib-docker
mountPath: /var/lib/docker
readOnly: false
{{- end }}
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
{{- if .Values.prod_environment }}
- name: airflow-logrotate
image: {{ .Values.images.tags.airflow }}
imagePullPolicy: {{ .Values.images.pull_policy }}
@ -214,13 +168,11 @@ spec:
volumeMounts:
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
{{- end }}
volumes:
- name: airflow-etc
configMap:
name: airflow-etc
defaultMode: 0444
{{- if .Values.prod_environment }}
- name: shipyard-etc
configMap:
name: shipyard-etc
@ -229,10 +181,6 @@ spec:
configMap:
name: airflow-bin
defaultMode: 0555
- name: pod-shipyard-share-airflow-dags
emptyDir: {}
- name: pod-shipyard-share-airflow-plugins
emptyDir: {}
volumeClaimTemplates:
- metadata:
name: airflow-logs
@ -242,21 +190,4 @@ spec:
requests:
storage: {{ .Values.volume_worker.size }}
storageClassName: {{ .Values.volume_worker.class_name }}
{{ else }}
- name: airflow-dags
hostPath:
path: {{ .Values.pod.mounts.dag_path }}
- name: airflow-plugins
hostPath:
path: {{ .Values.pod.mounts.plugin_path }}
- name: airflow-logs
hostPath:
path: {{ .Values.pod.mounts.log_path }}
- name: docker
hostPath:
path: /var/run
- name: pod-var-lib-docker
hostPath:
path: /var/lib/docker
{{- end }}
{{- end }}

6
charts/shipyard/values.yaml
View File

@ -15,9 +15,6 @@
# This file provides defaults for shipyard and airflow
# Indicate whether it is production or development environment
prod_environment: true
labels:
job:
node_selector_key: ucp-control-plane
@ -701,9 +698,6 @@ conf:
# End of Airflow config options
pod:
mounts:
dag_path: /home/ubuntu/workbench/dags
plugin_path: /home/ubuntu/workbench/plugins
log_path: /home/ubuntu/workbench/logs
airflow_scheduler:
# TODO: This is only used if the standalone scheduler is enabled.
airflow_scheduler:

16
images/airflow/Dockerfile
View File

@ -104,7 +104,7 @@ ENV PBR_VERSION 0.1a1
# Shipyard
#
# Shipyard provides core functionality used by the airflow plugins/operators
# Shipyard provides core functionality used by the Airflow plugins/operators
# Since Shipyard and Airflow are built together as images, this should prevent
# stale or out-of-date code between these parts.
# Shipyard requirements, source and installation
@ -115,6 +115,20 @@ COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
RUN cd /tmp/shipyard \
&& python3 setup.py install
# Note: The value for the dags and plugins directories that are sourced
# from the values.yaml of the Shipyard Helm chart need to align with these
# directories. If they do not, airflow will not find the intended dags and
# plugins.
#
# Note: In the case of building images using the provided Makefile, a test is
# run against the built-in dags provided with Airflow. Since there is no Helm
# chart to reconfigure the airflow.cfg with these directories, these dags and
# plugins are not known to Airflow during the image test.
#
# Copy the plugins and dags that will be used by this Airflow image:
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/
# Set work directory
USER airflow
WORKDIR ${AIRFLOW_HOME}