airship/pegleg
Code Issues Proposed changes

Airflow and Openstack modules upgrade

Browse Source 
This PS upgrades Airflow to 2.10.4 and Openstack
modules to 2024.1 Caracal versions

Change-Id: I55288b3fb6249b4659f12644409c43e93526263c
This commit is contained in:
Sergiy Markin 2024-12-17 19:32:49 +00:00
parent 387a09388c
commit ba9019e6d5
8 changed files with 106 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.zuul.yaml
View File

@ -254,7 +254,7 @@
commit: true commit: true
static: static:
- latest - latest
- airflow_2.10.2 - airflow_2.10.4
- secret: - secret:
name: airship_pegleg_quay_creds name: airship_pegleg_quay_creds

2
images/pegleg/Dockerfile.ubuntu_jammy
View File

@ -4,7 +4,7 @@
# 429 Too Many Requests - Server message: too many requests: # 429 Too Many Requests - Server message: too many requests:
# You have reached your pull rate limit. # You have reached your pull rate limit.
# You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit # You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit
ARG FROM=public.ecr.aws/ubuntu/ubuntu:jammy ARG FROM=public.ecr.aws/docker/library/ubuntu:jammy
FROM ${FROM} FROM ${FROM}
ARG CFSSLURL=https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 ARG CFSSLURL=https://pkg.cfssl.org/R1.2/cfssl_linux-amd64

10
pegleg/engine/lint.py
View File

@ -16,9 +16,9 @@ import logging
import os import os
import shutil import shutil
import textwrap import textwrap
import yaml
import click import click
import pkg_resources
from prettytable import PrettyTable from prettytable import PrettyTable
from pegleg import config from pegleg import config
@ -29,6 +29,7 @@ from pegleg.engine.errorcodes import REPOS_MISSING_DIRECTORIES_FLAG
from pegleg.engine.errorcodes import SCHEMA_STORAGE_POLICY_MISMATCH_FLAG from pegleg.engine.errorcodes import SCHEMA_STORAGE_POLICY_MISMATCH_FLAG
from pegleg.engine.errorcodes import SECRET_NOT_ENCRYPTED_POLICY from pegleg.engine.errorcodes import SECRET_NOT_ENCRYPTED_POLICY
from pegleg.engine import util from pegleg.engine import util
from importlib.resources import files
__all__ = ['full'] __all__ = ['full']
@ -367,7 +368,10 @@ def _expected_layer(filename):
def _load_schemas(): def _load_schemas():
schemas = {} schemas = {}
for key, filename in DECKHAND_SCHEMAS.items(): for key, filename in DECKHAND_SCHEMAS.items():
schemas[key] = util.files.slurp( filename_path = files('pegleg') / filename
pkg_resources.resource_filename('pegleg', filename)) filename_contents = filename_path.read_text(encoding='utf-8')
schemas[key] = yaml.safe_load(filename_contents)
return schemas return schemas

52
requirements-direct.txt
View File

@ -10,34 +10,34 @@ zipp
psycopg2-binary psycopg2-binary
# Airship dependencies # Airship dependencies
deckhand @ git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6#egg=deckhand deckhand @ git+https://opendev.org/airship/deckhand.git@4d500e48e880ea7f9a3582324c55c61373a855ea#egg=deckhand
shipyard-client @ git+https://opendev.org/airship/shipyard.git@e917c9dca721fcf3384c583c1c0d8fe9654a7987#egg=shipyard-client&subdirectory=src/bin/shipyard_client shipyard-client @ git+https://opendev.org/airship/shipyard.git@2fb62659c5a3eecafcf0d0dedaabf2a00fc5d88f#egg=shipyard-client&subdirectory=src/bin/shipyard_client
promenade @ git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66#egg=promenade promenade @ git+https://opendev.org/airship/promenade.git@e491bc4ca7a9764325ab4a2cf1ae10daf6a26efb#egg=promenade
# Openstack Antelope 2023.1 # Openstack Caracal 2024.1
# https://releases.openstack.org/antelope/index.html # https://releases.openstack.org/caracal/index.html
barbican==16.0.0 barbican==18.0.0
python-barbicanclient==5.5.0 python-barbicanclient==5.7.0
python-keystoneclient==5.1.0 python-keystoneclient==5.4.0
keystoneauth1==5.1.2 keystoneauth1==5.6.0
keystonemiddleware==10.2.0 keystonemiddleware==10.6.0
oslo.cache==3.3.1 oslo.cache==3.7.0
oslo.concurrency==5.1.1 oslo.concurrency==6.0.0
oslo.config==9.1.1 oslo.config==9.4.0
oslo.context==5.1.1 oslo.context==5.5.0
oslo.db==12.3.1 oslo.db==15.0.0
oslo.i18n==6.0.0 oslo.i18n==6.3.0
oslo.log==5.2.0 oslo.log==5.5.1
oslo.messaging==14.2.4 oslo.messaging==14.7.2
oslo.metrics==0.6.0 oslo.metrics==0.8.0
oslo.middleware==5.1.1 oslo.middleware==6.1.0
oslo.policy==4.1.1 oslo.policy==4.3.0
oslo.serialization==5.1.1 oslo.serialization==5.4.1
oslo.service==3.1.1 oslo.service==3.4.1
oslo.upgradecheck==2.1.1 oslo.upgradecheck==2.3.0
oslo.utils==6.1.0 oslo.utils==7.1.0
oslo.versionedobjects==3.1.0 oslo.versionedobjects==3.3.0

127
requirements-frozen.txt
View File

@ -1,81 +1,81 @@
alembic==1.13.2 alembic==1.14.0
amqp==5.2.0 amqp==5.3.1
arrow==1.3.0 arrow==1.3.0
attrs==24.2.0 attrs==24.2.0
autopage==0.5.2 autopage==0.5.2
barbican==16.0.0 barbican==18.0.0
bcrypt==4.2.0 bcrypt==4.2.1
Beaker==1.13.0 Beaker==1.13.0
cachetools==5.5.0 cachetools==5.5.0
castellan==5.1.1 castellan==5.2.0
certifi==2024.8.30 certifi==2024.8.30
cffi==1.17.1 cffi==1.17.1
charset-normalizer==3.3.2 charset-normalizer==3.4.0
click==8.1.7 click==8.1.7
click-default-group==1.2.4 click-default-group==1.2.4
cliff==4.7.0 cliff==4.8.0
cmd2==2.4.3 cmd2==2.5.8
cryptography==42.0.8 cryptography==42.0.8
debtcollector==3.0.0 debtcollector==3.0.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6 Deckhand @ git+https://opendev.org/airship/deckhand.git@4d500e48e880ea7f9a3582324c55c61373a855ea
decorator==5.1.1 decorator==5.1.1
deepdiff==8.0.1 deepdiff==8.1.1
dnspython==2.6.1 dnspython==2.7.0
docker==7.1.0 docker==7.1.0
dogpile.cache==1.3.3 dogpile.cache==1.3.3
eventlet==0.37.0 eventlet==0.38.1
falcon==3.1.3 falcon==4.0.2
fasteners==0.19 fasteners==0.19
fixtures==4.1.0 fixtures==4.1.0
Flask==2.2.5 Flask==2.2.5
futurist==3.0.0 futurist==3.0.0
gitdb==4.0.11 gitdb==4.0.11
GitPython==3.1.43 GitPython==3.1.43
google-auth==2.34.0 google-auth==2.36.0
greenlet==3.1.0 greenlet==3.1.1
html5lib==0.9999999 html5lib==0.9999999
httpexceptor==1.4.0 httpexceptor==1.4.0
idna==3.10 idna==3.10
iso8601==2.1.0 iso8601==2.1.0
itsdangerous==2.2.0 itsdangerous==2.2.0
Jinja2==3.1.4 Jinja2==3.1.4
jsonpath-ng==1.6.1 jsonpath-ng==1.7.0
jsonpickle==3.3.0 jsonpickle==3.4.2
jsonschema==4.23.0 jsonschema==4.23.0
jsonschema-specifications==2023.12.1 jsonschema-specifications==2023.12.1
keystoneauth1==5.1.2 keystoneauth1==5.6.0
keystonemiddleware==10.2.0 keystonemiddleware==10.6.0
kombu==5.4.1 kombu==5.4.2
kubernetes==30.1.0 kubernetes==30.1.0
ldap3==2.9.1 ldap3==2.9.1
logutils==0.3.5 logutils==0.3.5
Mako==1.3.5 Mako==1.3.8
MarkupSafe==2.1.5 MarkupSafe==3.0.2
microversion-parse==2.0.0 microversion-parse==2.0.0
msgpack==1.1.0 msgpack==1.1.0
netaddr==1.3.0 netaddr==1.3.0
netifaces==0.11.0 netifaces==0.11.0
networkx==3.3 networkx==3.4.2
oauthlib==3.2.2 oauthlib==3.2.2
orderly-set==5.2.2 orderly-set==5.2.3
os-service-types==1.7.0 os-service-types==1.7.0
oslo.cache==3.3.1 oslo.cache==3.7.0
oslo.concurrency==5.1.1 oslo.concurrency==6.0.0
oslo.config==9.1.1 oslo.config==9.4.0
oslo.context==5.1.1 oslo.context==5.5.0
oslo.db==12.3.1 oslo.db==15.0.0
oslo.i18n==6.0.0 oslo.i18n==6.3.0
oslo.log==5.2.0 oslo.log==5.5.1
oslo.messaging==14.2.4 oslo.messaging==14.7.2
oslo.metrics==0.6.0 oslo.metrics==0.8.0
oslo.middleware==5.1.1 oslo.middleware==6.1.0
oslo.policy==4.1.1 oslo.policy==4.3.0
oslo.serialization==5.1.1 oslo.serialization==5.4.1
oslo.service==3.1.1 oslo.service==3.4.1
oslo.upgradecheck==2.1.1 oslo.upgradecheck==2.3.0
oslo.utils==6.1.0 oslo.utils==7.1.0
oslo.versionedobjects==3.1.0 oslo.versionedobjects==3.3.0
packaging==24.1 packaging==24.2
Paste==3.10.1 Paste==3.10.1
PasteDeploy==3.1.0 PasteDeploy==3.1.0
PasteScript==3.6.0 PasteScript==3.6.0
@ -83,22 +83,23 @@ pbr==6.1.0
pecan==1.5.1 pecan==1.5.1
pip==24.1 pip==24.1
ply==3.11 ply==3.11
prettytable==3.11.0 prettytable==3.12.0
promenade @ git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66 promenade @ git+https://opendev.org/airship/promenade.git@e491bc4ca7a9764325ab4a2cf1ae10daf6a26efb
prometheus_client==0.20.0 prometheus_client==0.21.1
psycopg2-binary==2.9.9 psycopg2-binary==2.9.10
PTable==0.9.2 PTable==0.9.2
pyasn1==0.6.1 pyasn1==0.6.1
pyasn1_modules==0.4.0 pyasn1_modules==0.4.0
pycadf==3.1.1 pycadf==4.0.0
pycparser==2.22 pycparser==2.22
PyJWT==2.10.1
pylibyaml==0.1.0 pylibyaml==0.1.0
pyOpenSSL==24.2.1 pyOpenSSL==24.3.0
pyparsing==3.1.4 pyparsing==3.2.0
pyperclip==1.9.0 pyperclip==1.9.0
python-barbicanclient==5.5.0 python-barbicanclient==5.7.0
python-dateutil==2.9.0.post0 python-dateutil==2.9.0.post0
python-keystoneclient==5.1.0 python-keystoneclient==5.4.0
python-memcached==1.62 python-memcached==1.62
python-mimeparse==2.0.0 python-mimeparse==2.0.0
pytz==2024.2 pytz==2024.2
@ -110,35 +111,33 @@ requests-oauthlib==1.3.1
resolver==0.2.1 resolver==0.2.1
rfc3986==2.0.0 rfc3986==2.0.0
Routes==2.5.1 Routes==2.5.1
rpds-py==0.20.0 rpds-py==0.22.3
rsa==4.9 rsa==4.9
selector==0.10.1 selector==0.10.1
setuptools==70.1.0 setuptools==70.1.0
shipyard_client @ git+https://opendev.org/airship/shipyard.git@e917c9dca721fcf3384c583c1c0d8fe9654a7987#subdirectory=src/bin/shipyard_client shipyard_client @ git+https://opendev.org/airship/shipyard.git@2fb62659c5a3eecafcf0d0dedaabf2a00fc5d88f#subdirectory=src/bin/shipyard_client
simplejson==3.19.3 simplejson==3.19.3
six==1.16.0 six==1.17.0
smmap==5.0.1 smmap==5.0.1
SQLAlchemy==1.4.54 SQLAlchemy==1.4.54
sqlalchemy-migrate==0.13.0
sqlparse==0.5.1
statsd==4.0.1 statsd==4.0.1
stevedore==5.3.0 stevedore==5.4.0
Tempita==0.5.2
testresources==2.0.1 testresources==2.0.1
testscenarios==0.5.0 testscenarios==0.5.0
testtools==2.7.2 testtools==2.7.2
tiddlyweb==2.4.3 tiddlyweb==2.4.3
types-python-dateutil==2.9.0.20240906 types-python-dateutil==2.9.0.20241206
typing_extensions==4.12.2 typing_extensions==4.12.2
urllib3==2.2.2 tzdata==2024.2
uWSGI==2.0.27 urllib3==2.2.3
uWSGI==2.0.28
vine==5.1.0 vine==5.1.0
wcwidth==0.2.13 wcwidth==0.2.13
WebOb==1.8.8 WebOb==1.8.9
websocket-client==1.8.0 websocket-client==1.8.0
Werkzeug==2.2.3 Werkzeug==2.2.3
wheel==0.43.0 wheel==0.43.0
wrapt==1.16.0 wrapt==1.17.0
xattr==0.10.1 xattr==0.10.1
yappi==1.6.0 yappi==1.6.10
zipp==3.20.2 zipp==3.21.0

8
test-requirements.txt
View File

@ -1,12 +1,12 @@
bandit~=1.6.0 bandit~=1.6.0
pytest pytest==7.4.0
pytest-cov pytest-cov==4.1.0
testfixtures testfixtures
pytest-xdist pytest-xdist
requests>=2.20.0 requests>=2.20.0
urllib3>=1.24.3 urllib3>=1.24.3
chardet>=3.0.4 chardet>=3.0.4
flake8 flake8
yapf~=0.28.0 yapf
hacking>=1.1.0 hacking>=1.1.0
flake8-import-order>=0.18.1 flake8-import-order

2
tox.ini
View File

@ -99,7 +99,7 @@ allowlist_externals=
sh sh
deps= deps=
-r{toxinidir}/requirements-direct.txt -r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.10.txt
commands= commands=
rm -f {toxinidir}/requirements-frozen.txt rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'pegleg|pyinotify|pkg-resources' > requirements-frozen.txt" sh -c "pip freeze --all | grep -vE 'pegleg|pyinotify|pkg-resources' > requirements-frozen.txt"

6
vcs-requirements.env
View File

@ -1,3 +1,3 @@
DECKHAND_VERSION=commit/be9f97b846b1edc6b74777e5e639aa9edce8cfa6 DECKHAND_VERSION=commit/4d500e48e880ea7f9a3582324c55c61373a855ea
SHIPYARD_VERSION=commit/e917c9dca721fcf3384c583c1c0d8fe9654a7987 SHIPYARD_VERSION=commit/2fb62659c5a3eecafcf0d0dedaabf2a00fc5d88f
PROMENADE_VERSION=commit/ac573b9fb53fd543172c50bbbcdfac3656fb8e66 PROMENADE_VERSION=commit/e491bc4ca7a9764325ab4a2cf1ae10daf6a26efb