airship/shipyard
Code Issues Proposed changes

Move Airflow image dependencies to requirements.txt

Browse Source 
This reduces the number of layers built by the Dockerfile and puts
more of hte dependency management into a common place.

Change-Id: I416ac11749f3fe8803919994e807f46d64493b5a
This commit is contained in:
Bryan Strassner 2018-04-16 15:50:41 -05:00 committed by Anthony Lin
parent 6b0100ab00
commit 83d91689aa
3 changed files with 21 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
images/airflow/Dockerfile
View File

@ -67,33 +67,22 @@ RUN set -ex && \
/usr/share/doc \ /usr/share/doc \
/usr/share/doc-base /usr/share/doc-base
# Copy dependency requirements # Things that change mostly infrequently
COPY ./requirements.txt /tmp/ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
RUN pip3 install -r /tmp/requirements.txt && curl -L -o /usr/local/bin/kubectl \
https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
# Note - removing snakebite (python 2 vs. 3). See:
# https://github.com/puckel/docker-airflow/issues/77
RUN pip3 uninstall -y snakebite || true
# Install Armada, DeckHand and DryDock Client Libraries
RUN pip3 install -e git://github.com/att-comdev/armada.git@7a2ba22ab12a3f1f180b6af4085972ba44853377#egg=armada
RUN pip3 install -e git://github.com/att-comdev/deckhand.git@3cdf3d2d896d43c6e3bc26170522c3eee0d7158f#egg=deckhand
RUN pip3 install -e git://github.com/att-comdev/drydock.git@42aa3c486ee4c495c2377d31481df5ab681f84f2#egg=drydock_provisioner
# Create airflow user
RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow
# Download and install kubectl
RUN curl -L -o /usr/local/bin/kubectl \
https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
&& chmod +x /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl
# Copy entrypoint.sh, airflow_start_service.sh and # Dependency requirements
# airflow_logrotate.sh # Note - removing snakebite (python 2 vs. 3). See:
COPY script/entrypoint.sh ${AIRFLOW_HOME}/entrypoint.sh # https://github.com/puckel/docker-airflow/issues/77
COPY script/airflow_start_service.sh ${AIRFLOW_HOME}/airflow_start_service.sh COPY ./requirements.txt /tmp/
COPY script/airflow_logrotate.sh ${AIRFLOW_HOME}/airflow_logrotate.sh RUN pip3 install -r /tmp/requirements.txt \
COPY script/upgrade_airflow_worker.sh ${AIRFLOW_HOME}/upgrade_airflow_worker.sh && pip3 uninstall -y snakebite || true
# Copy scripts used in the container:
# entrypoint.sh, airflow_start_service.sh and airflow_logrotate.sh
COPY script/*.sh ${AIRFLOW_HOME}/
# Change permissions # Change permissions
RUN chown -R airflow: ${AIRFLOW_HOME} RUN chown -R airflow: ${AIRFLOW_HOME}

5
images/airflow/requirements.txt
View File

@ -22,3 +22,8 @@ apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.9.0
python-openstackclient==3.11.0 python-openstackclient==3.11.0
sphinx>=1.6.2 sphinx>=1.6.2
sphinx_rtd_theme==0.2.4 sphinx_rtd_theme==0.2.4
# Dependencies for other UCP components
git+https://github.com/att-comdev/deckhand.git@3cdf3d2d896d43c6e3bc26170522c3eee0d7158f#egg=deckhand
git+https://github.com/att-comdev/drydock.git@42aa3c486ee4c495c2377d31481df5ab681f84f2#egg=drydock_provisioner
git+https://github.com/att-comdev/armada.git@7a2ba22ab12a3f1f180b6af4085972ba44853377#egg=armada

3
test-requirements.txt
View File

@ -10,7 +10,8 @@ apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.9.0
git+https://github.com/att-comdev/deckhand.git@3cdf3d2d896d43c6e3bc26170522c3eee0d7158f#egg=deckhand git+https://github.com/att-comdev/deckhand.git@3cdf3d2d896d43c6e3bc26170522c3eee0d7158f#egg=deckhand
git+https://github.com/att-comdev/drydock.git@42aa3c486ee4c495c2377d31481df5ab681f84f2#egg=drydock_provisioner git+https://github.com/att-comdev/drydock.git@42aa3c486ee4c495c2377d31481df5ab681f84f2#egg=drydock_provisioner
git+https://github.com/att-comdev/armada.git@7a2ba22ab12a3f1f180b6af4085972ba44853377#egg=armada git+https://github.com/att-comdev/armada.git@7a2ba22ab12a3f1f180b6af4085972ba44853377#egg=armada
# TODO(bryan-strassner) Pin to Dockerfile version for airflow when added there # TODO(bryan-strassner) Pin to version for airflow when added to the
# requirements.txt in the airflow images directory
git+https://github.com/att-comdev/promenade.git@master#egg=promenade git+https://github.com/att-comdev/promenade.git@master#egg=promenade