airship/shipyard
Code Issues Proposed changes

Update pip package versions in preparation of pip 20.3

Browse Source 
When pip is upgraded to 20.3, the pip dependency resolver is much more
strict and will no longer install a combination of packages that is mutually
inconsistent[0].

These changes account for the fact that Shipyard imports Armada, Drydock,
Promenade, and Deckhand. Having said that, with pip 20.3, the pip
packages amongst those projects cannot conflict. A follow-up change may
be needed if more conflicts are found.

Change-Id: Idd9ea0d57b5be063b133036cfc9ebaa69956f4fc
This commit is contained in:
Rick Bartra 2020-09-28 15:14:25 -04:00 committed by Rick Bartra
parent fb1c281b83
commit 35e5a7f796
8 changed files with 64 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
images/airflow/Dockerfile.opensuse_15
View File

@ -40,7 +40,10 @@ EXPOSE $WORKER_PORT
# Set ARG for usage during build
ARG AIRFLOW_HOME=/usr/local/airflow
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.5"
# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
# version of celery incompatibile with the version of kombu needed by other
# Airship components
ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5"
ARG ctx_base=src/bin
# Kubectl version

5
images/airflow/Dockerfile.ubuntu_bionic
View File

@ -40,7 +40,10 @@ EXPOSE $WORKER_PORT
# Set ARG for usage during build
ARG AIRFLOW_HOME=/usr/local/airflow
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.5"
# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
# version of celery incompatibile with the version of kombu needed by other
# Airship components
ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5"
ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin

5
images/airflow/Dockerfile.ubuntu_xenial
View File

@ -40,7 +40,10 @@ EXPOSE $WORKER_PORT
# Set ARG for usage during build
ARG AIRFLOW_HOME=/usr/local/airflow
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.5"
# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
# version of celery incompatibile with the version of kombu needed by other
# Airship components
ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5"
ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin

21
images/airflow/requirements.txt
View File

@ -12,22 +12,33 @@
# See the License for the specific language governing permissions and
# limitations under the License.
celery==4.3
pytz==2018.5
python-dateutil==2.8.1
pyOpenSSL==18.0.0
ndg-httpsclient==0.5.1
pyasn1==0.4.4
oslo.utils==3.42.1
oslo.config==7.0.0
oslo.serialization==2.29.2
pyasn1<0.5.0,>=0.4.6
psycopg2-binary==2.8.4
docker==3.5.0
docker==3.7.2
# Airflow is now installed in the Dockerfile directory to allow for
# overriding where it is sourced from
python-openstackclient==3.16.1
python-keystoneclient==3.22.0
kubernetes>=6.0.0
# Need to lock marshmellow-sqlalchemy and tabulate for compatibility issues
marshmallow-sqlalchemy==0.18.0
WTForms~=2.2.1
tabulate==0.8.03
pbr==5.4.5
setuptools>=40.4.3
six>=1.15.0
urllib3==1.25.9
vine<5.0.0a1,>=1.1.3
# Dependencies for other UCP components
git+https://opendev.org/airship/deckhand.git@e7ba6828a0a1ca27fae596f6e0ee5a857f28001d#egg=deckhand
git+https://opendev.org/airship/drydock.git@586bcf8ebed430b4de82edd9a527566ed39704b7#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@af8a9ffd0873c2fbc915794e235dbd357f2adab1#egg=armada
git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65edada7d386cd55d2a#egg=deckhand
git+https://opendev.org/airship/drydock.git@96db568d4b28f5f163efbf606854243614a6c821#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@0c3aff1ba1f0e836e3dd56de81b2ee560e11c179#egg=armada

38
src/bin/shipyard_airflow/requirements.txt
View File

@ -14,27 +14,37 @@
# API requirements
alembic==1.0.0
alembic==1.0.1
arrow==0.12.1
celery~=4.4.2
celery==4.3
kombu<4.7,>=4.6.10
vine<5.0.0a1,>=1.1.3
configparser==3.5.0
cryptography==2.3
cryptography>=2.7
falcon==1.4.1
jsonschema==2.6.0
jsonschema>=3.0.1<4
grpcio>=1.16.0
keystoneauth1==3.11.0
keystonemiddleware==5.2.0
networkx==2.1 # common/deployment_group
oslo.config==6.4.0
oslo.policy==1.38.1
keystoneauth1>=3.18.0
keystonemiddleware==5.3.0
networkx==2.2 # common/deployment_group
oslo.config==7.0.0
oslo.policy==1.40.1
PasteDeploy==1.5.2
psycopg2-binary==2.8.4
pylibyaml~=0.1
python-dateutil==2.7.3
oslo.cache==1.38.1
oslo.log==3.45.2
oslo.utils==3.42.1
oslo.serialization==2.29.2
pbr==5.4.5
python-dateutil==2.8.1
python-memcached==1.59
requests==2.20.0
python-keystoneclient==3.22.0
requests!=2.20.0,>=2.14.2
setuptools==40.4.3
SQLAlchemy==1.3.15
ulid==1.1
urllib3==1.25.9
uwsgi~=2.0.19.1
# To support profiling in non-prod
@ -45,6 +55,6 @@ marshmallow-sqlalchemy==0.18.0
tabulate==0.8.03
# Dependencies for other UCP components
git+https://opendev.org/airship/deckhand.git@e7ba6828a0a1ca27fae596f6e0ee5a857f28001d#egg=deckhand
git+https://opendev.org/airship/drydock.git@586bcf8ebed430b4de82edd9a527566ed39704b7#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@af8a9ffd0873c2fbc915794e235dbd357f2adab1#egg=armada
git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65edada7d386cd55d2a#egg=deckhand
git+https://opendev.org/airship/drydock.git@96db568d4b28f5f163efbf606854243614a6c821#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@0c3aff1ba1f0e836e3dd56de81b2ee560e11c179#egg=armada

11
src/bin/shipyard_airflow/test-requirements.txt
View File

@ -5,14 +5,21 @@ pytest==3.5.0
pytest-cov==2.5.1
responses==0.10.2
testfixtures==5.1.1
apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1
# Had to upgrade apache-airflow as 1.10.1 requires a version of jinja too old
# for promenade
apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.2
# install postgres seperately as apache-airflow 1.10.2 forces postgres to use
# psycopg2 instead of psycopg2-binary which requires additional apt packages
# to be installed, i.e. postgres-devel. Otherwise the following error is seen:
# Error: pg_config executable not found. Only seems to be an issue in xenial
postgres>=3.0.0
# TODO(bryan-strassner) Pin to version for airflow when added to the
# requirements.txt in the airflow images directory
git+https://opendev.org/airship/promenade.git@master#egg=promenade
# Linting
flake8>=3.3.0
flake8<3.8.0,>=3.6.0
# Security scanning
bandit>=1.5.0 # Apache-2.0

4
src/bin/shipyard_client/requirements.txt
View File

@ -17,8 +17,8 @@
arrow==0.12.1
click==6.7
click-default-group==1.2
keystoneauth1==3.11.1
requests==2.20.0
keystoneauth1>=3.18.0
requests!=2.20.0,>=2.14.2
PTable==0.9.2
pylibyaml==0.1.0
pyyaml==5.1

2
src/bin/shipyard_client/test-requirements.txt
View File

@ -7,7 +7,7 @@ responses==0.10.2
testfixtures==5.1.1
# Linting
flake8>=3.3.0
flake8<3.8.0,>=3.6.0
# Security scanning
bandit>=1.1.0 # Apache-2.0