Update pip package versions in preparation of pip 20.3

When pip is upgraded to 20.3, the pip dependency resolver is much more
strict and will no longer install a combination of packages that is mutually
inconsistent[0].

These changes account for the fact that Shipyard imports Armada, Drydock,
Promenade, and Deckhand. Having said that, with pip 20.3, the pip
packages amongst those projects cannot conflict. A follow-up change may
be needed if more conflicts are found.

Change-Id: Idd9ea0d57b5be063b133036cfc9ebaa69956f4fc

images/airflow/Dockerfile.opensuse_15

@@ -40,7 +40,10 @@ EXPOSE $WORKER_PORT
 
 # Set ARG for usage during build
 ARG AIRFLOW_HOME=/usr/local/airflow
-ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.5"
+# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
+# version of celery incompatibile with the version of kombu needed by other
+# Airship components
+ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5"
 ARG ctx_base=src/bin
 
 # Kubectl version

images/airflow/Dockerfile.ubuntu_bionic

@@ -40,7 +40,10 @@ EXPOSE $WORKER_PORT
 
 # Set ARG for usage during build
 ARG AIRFLOW_HOME=/usr/local/airflow
-ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.5"
+# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
+# version of celery incompatibile with the version of kombu needed by other
+# Airship components
+ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5"
 ARG DEBIAN_FRONTEND=noninteractive
 ARG ctx_base=src/bin
 

images/airflow/Dockerfile.ubuntu_xenial

@@ -40,7 +40,10 @@ EXPOSE $WORKER_PORT
 
 # Set ARG for usage during build
 ARG AIRFLOW_HOME=/usr/local/airflow
-ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.5"
+# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
+# version of celery incompatibile with the version of kombu needed by other
+# Airship components
+ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5"
 ARG DEBIAN_FRONTEND=noninteractive
 ARG ctx_base=src/bin
 

images/airflow/requirements.txt

@@ -12,22 +12,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+celery==4.3
 pytz==2018.5
+python-dateutil==2.8.1
 pyOpenSSL==18.0.0
 ndg-httpsclient==0.5.1
-pyasn1==0.4.4
+oslo.utils==3.42.1
+oslo.config==7.0.0
+oslo.serialization==2.29.2
+pyasn1<0.5.0,>=0.4.6
 psycopg2-binary==2.8.4
-docker==3.5.0
+docker==3.7.2
 # Airflow is now installed in the Dockerfile directory to allow for
 # overriding where it is sourced from
 python-openstackclient==3.16.1
+python-keystoneclient==3.22.0
 kubernetes>=6.0.0
 # Need to lock marshmellow-sqlalchemy and tabulate for compatibility issues
 marshmallow-sqlalchemy==0.18.0
 WTForms~=2.2.1
 tabulate==0.8.03
+pbr==5.4.5
+setuptools>=40.4.3
+six>=1.15.0
+urllib3==1.25.9
+vine<5.0.0a1,>=1.1.3
 
 # Dependencies for other UCP components
-git+https://opendev.org/airship/deckhand.git@e7ba6828a0a1ca27fae596f6e0ee5a857f28001d#egg=deckhand
+git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65edada7d386cd55d2a#egg=deckhand
-git+https://opendev.org/airship/drydock.git@586bcf8ebed430b4de82edd9a527566ed39704b7#egg=drydock_provisioner&subdirectory=python
+git+https://opendev.org/airship/drydock.git@96db568d4b28f5f163efbf606854243614a6c821#egg=drydock_provisioner&subdirectory=python
-git+https://opendev.org/airship/armada.git@af8a9ffd0873c2fbc915794e235dbd357f2adab1#egg=armada
+git+https://opendev.org/airship/armada.git@0c3aff1ba1f0e836e3dd56de81b2ee560e11c179#egg=armada

src/bin/shipyard_airflow/requirements.txt

@@ -14,27 +14,37 @@
 
 # API requirements
 
-alembic==1.0.0
+alembic==1.0.1
 arrow==0.12.1
-celery~=4.4.2
+celery==4.3
+kombu<4.7,>=4.6.10
+vine<5.0.0a1,>=1.1.3
 configparser==3.5.0
-cryptography==2.3
+cryptography>=2.7
 falcon==1.4.1
-jsonschema==2.6.0
+jsonschema>=3.0.1<4
 grpcio>=1.16.0
-keystoneauth1==3.11.0
+keystoneauth1>=3.18.0
-keystonemiddleware==5.2.0
+keystonemiddleware==5.3.0
-networkx==2.1 # common/deployment_group
+networkx==2.2 # common/deployment_group
-oslo.config==6.4.0
+oslo.config==7.0.0
-oslo.policy==1.38.1
+oslo.policy==1.40.1
 PasteDeploy==1.5.2
 psycopg2-binary==2.8.4
 pylibyaml~=0.1
-python-dateutil==2.7.3
+oslo.cache==1.38.1
+oslo.log==3.45.2
+oslo.utils==3.42.1
+oslo.serialization==2.29.2
+pbr==5.4.5
+python-dateutil==2.8.1
 python-memcached==1.59
-requests==2.20.0
+python-keystoneclient==3.22.0
+requests!=2.20.0,>=2.14.2
+setuptools==40.4.3
 SQLAlchemy==1.3.15
 ulid==1.1
+urllib3==1.25.9
 uwsgi~=2.0.19.1
 
 # To support profiling in non-prod
@@ -45,6 +55,6 @@ marshmallow-sqlalchemy==0.18.0
 tabulate==0.8.03
 
 # Dependencies for other UCP components
-git+https://opendev.org/airship/deckhand.git@e7ba6828a0a1ca27fae596f6e0ee5a857f28001d#egg=deckhand
+git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65edada7d386cd55d2a#egg=deckhand
-git+https://opendev.org/airship/drydock.git@586bcf8ebed430b4de82edd9a527566ed39704b7#egg=drydock_provisioner&subdirectory=python
+git+https://opendev.org/airship/drydock.git@96db568d4b28f5f163efbf606854243614a6c821#egg=drydock_provisioner&subdirectory=python
-git+https://opendev.org/airship/armada.git@af8a9ffd0873c2fbc915794e235dbd357f2adab1#egg=armada
+git+https://opendev.org/airship/armada.git@0c3aff1ba1f0e836e3dd56de81b2ee560e11c179#egg=armada

src/bin/shipyard_airflow/test-requirements.txt

@@ -5,14 +5,21 @@ pytest==3.5.0
 pytest-cov==2.5.1
 responses==0.10.2
 testfixtures==5.1.1
-apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1
+# Had to upgrade apache-airflow as 1.10.1 requires a version of jinja too old
+# for promenade
+apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.2
+# install postgres seperately as apache-airflow 1.10.2 forces postgres to use
+# psycopg2 instead of psycopg2-binary which requires additional apt packages
+# to be installed, i.e. postgres-devel. Otherwise the following error is seen:
+# Error: pg_config executable not found. Only seems to be an issue in xenial
+postgres>=3.0.0
 
 # TODO(bryan-strassner) Pin to version for airflow when added to the
 # requirements.txt in the airflow images directory
 git+https://opendev.org/airship/promenade.git@master#egg=promenade
 
 # Linting
-flake8>=3.3.0
+flake8<3.8.0,>=3.6.0
 
 # Security scanning
 bandit>=1.5.0 # Apache-2.0

src/bin/shipyard_client/requirements.txt

@@ -17,8 +17,8 @@
 arrow==0.12.1
 click==6.7
 click-default-group==1.2
-keystoneauth1==3.11.1
+keystoneauth1>=3.18.0
-requests==2.20.0
+requests!=2.20.0,>=2.14.2
 PTable==0.9.2
 pylibyaml==0.1.0
 pyyaml==5.1

src/bin/shipyard_client/test-requirements.txt

@@ -7,7 +7,7 @@ responses==0.10.2
 testfixtures==5.1.1
 
 # Linting
-flake8>=3.3.0
+flake8<3.8.0,>=3.6.0
 
 # Security scanning
 bandit>=1.1.0 # Apache-2.0