(airflow) Configure source of Airflow pkg

- Allow a Docker build-arg to specify the source of the Airflow
  python package in any format supported by pip

Change-Id: Ifa2dd62d91570035cff91ff07868b0fcf659add8

Makefile

@@ -31,6 +31,7 @@ PROXY                      ?= http://proxy.foo.com:8000
 NO_PROXY                   ?= localhost,127.0.0.1,.svc.cluster.local
 USE_PROXY                  ?= false
 
+AIRFLOW_SRC                ?=
 DISTRO_BASE_IMAGE          ?=
 DISTRO                     ?= ubuntu_xenial
 
@@ -83,6 +84,9 @@ run:
 	USE_PROXY=$(USE_PROXY) PROXY=$(PROXY) $(SCRIPT) $(IMAGE)
 
 _BASE_IMAGE_ARG := $(if $(DISTRO_BASE_IMAGE),--build-arg FROM="${DISTRO_BASE_IMAGE}" ,)
+ifeq ($(IMAGE_NAME), airflow)
+	_AIRFLOW_SRC_ARG := $(if $(AIRFLOW_SRC),--build-arg AIRFLOW_SRC="${AIRFLOW_SRC}" ,)
+endif
 
 .PHONY: build
 build:
@@ -93,6 +97,7 @@ ifeq ($(USE_PROXY), true)
 		--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
 		-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
 		$(_BASE_IMAGE_ARG) \
+		$(_AIRFLOW_SRC_ARG) \
 		--build-arg http_proxy=$(PROXY) \
 		--build-arg https_proxy=$(PROXY) \
 		--build-arg HTTP_PROXY=$(PROXY) \
@@ -107,6 +112,7 @@ else
 		--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
 		-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
 		$(_BASE_IMAGE_ARG) \
+		$(_AIRFLOW_SRC_ARG) \
 		--build-arg ctx_base=$(BUILD_CTX) .
 endif
 ifeq ($(PUSH_IMAGE), true)

images/airflow/Dockerfile.opensuse_15

@@ -40,6 +40,7 @@ EXPOSE $WORKER_PORT
 
 # Set ARG for usage during build
 ARG AIRFLOW_HOME=/usr/local/airflow
+ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1"
 ARG ctx_base=src/bin
 
 # Kubectl version
@@ -110,7 +111,8 @@ ENV PBR_VERSION 0.1a1
 # stale or out-of-date code between these parts.
 # Shipyard requirements, source and installation
 COPY ${ctx_base}/shipyard_airflow/requirements.txt /tmp/api_requirements.txt
-RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
+RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir \
+    && pip3 install $AIRFLOW_SRC --no-cache-dir
 
 COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
 RUN cd /tmp/shipyard \

images/airflow/Dockerfile.ubuntu_xenial

@@ -40,6 +40,7 @@ EXPOSE $WORKER_PORT
 
 # Set ARG for usage during build
 ARG AIRFLOW_HOME=/usr/local/airflow
+ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1"
 ARG DEBIAN_FRONTEND=noninteractive
 ARG ctx_base=src/bin
 
@@ -85,8 +86,10 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
 # Dependency requirements
 # Note - removing snakebite (python 2 vs. 3). See:
 #    https://github.com/puckel/docker-airflow/issues/77
+# Install Airflow directly to allow overriding source
 COPY images/airflow/requirements.txt /tmp/
 RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
+    && pip3 install $AIRFLOW_SRC --no-cache-dir \
     && pip3 uninstall -y snakebite || true
 
 # Copy scripts used in the container:

images/airflow/requirements.txt

@@ -18,9 +18,11 @@ ndg-httpsclient==0.5.1
 pyasn1==0.4.4
 psycopg2==2.7.5
 docker==3.5.0
-apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1
+# Airflow is now installed in the Dockerfile directory to allow for
+# overriding where it is sourced from
 python-openstackclient==3.16.1
 kubernetes>=6.0.0
+marshmallow-sqlalchemy==0.18.0
 
 # Dependencies for other UCP components
 git+https://git.openstack.org/openstack/airship-deckhand@a76ffb66ae809c19281a6cf5c9414ed197d249b7#egg=deckhand

src/bin/shipyard_airflow/requirements.txt

@@ -36,7 +36,8 @@ ulid==1.1
 uwsgi==2.0.17
 
 # To support profiling in non-prod
-Werkzeug==0.14.1
+Werkzeug>=0.15.0
+marshmallow-sqlalchemy==0.18.0
 
 # Dependencies for other UCP components
 git+https://git.openstack.org/openstack/airship-deckhand@a76ffb66ae809c19281a6cf5c9414ed197d249b7#egg=deckhand