airship/shipyard
Code Issues Proposed changes

Fix deprecated code

Browse Source 
This PS replaces deprecared module pkg_resources,
also fixes some unit tests and fixes the schema
validation by adding specific schema draft to
choose in order to prevent the processor to fall
back to use the latest draft that may potentially
cause issues.

Also this PS bumps kubectl version to 1.32.0 due
to CVE and switched base ubuntu image repo to
quay.io/airshipit

Change-Id: Ie40f179eac83fde4d828e6f63a9c03e473eb3b15
This commit is contained in:
Sergiy Markin 2024-12-23 06:03:19 +00:00
parent 2fb62659c5
commit 86b7d03403
8 changed files with 96 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
images/airflow/Dockerfile.ubuntu_jammy
View File

@ -19,7 +19,7 @@
# 429 Too Many Requests - Server message: too many requests:
# You have reached your pull rate limit.
# You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit
ARG FROM=public.ecr.aws/docker/library/ubuntu:jammy
ARG FROM=quay.io/airshipit/ubuntu:jammy
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
@ -55,7 +55,7 @@ ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin
# Kubectl version
ARG KUBECTL_VERSION=1.31.0
ARG KUBECTL_VERSION=1.32.0
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
ENV AIRFLOW_HOME=${AIRFLOW_HOME}

2
images/shipyard/Dockerfile.ubuntu_jammy
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
ARG FROM=ubuntu:jammy
ARG FROM=quay.io/airshipit/ubuntu:jammy
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \

8
src/bin/shipyard_airflow/requirements-direct.txt
View File

@ -59,7 +59,7 @@ oslo.versionedobjects==3.3.0
# Airship dependencies
git+https://opendev.org/airship/deckhand.git@4d500e48e880ea7f9a3582324c55c61373a855ea#egg=deckhand
git+https://opendev.org/airship/drydock.git@ffcd51e00249198f1e2d40f3e83c8c1c338df3ad#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@af4bf814f3a44e9ccee7c9fd8e4be34d943e013d#egg=armada
git+https://opendev.org/airship/promenade.git@817ead27e991ff3fbed99ea1cb08aad5674d7167#egg=promenade
git+https://opendev.org/airship/deckhand.git@211b2d20c080ec806d196f3ec6b0d0472b00fa1b#egg=deckhand
git+https://opendev.org/airship/drydock.git@108a3c1ee4666df3a6fb4439df245525a870d1d9#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@963e23bb9323ea30472c54adc32e5c9eb7bf0506#egg=armada
git+https://opendev.org/airship/promenade.git@40fd108fd976c19bc142d10cc52b24d48de37100#egg=promenade

8
src/bin/shipyard_airflow/requirements-frozen.txt
View File

@ -21,7 +21,7 @@ apache-airflow-providers-smtp==1.8.1
apache-airflow-providers-sqlite==3.9.1
apispec==6.8.0
argcomplete==3.5.2
Armada @ git+https://opendev.org/airship/armada.git@af4bf814f3a44e9ccee7c9fd8e4be34d943e013d
Armada @ git+https://opendev.org/airship/armada.git@963e23bb9323ea30472c54adc32e5c9eb7bf0506
arrow==1.3.0
asgiref==3.8.1
async-timeout==5.0.1
@ -55,7 +55,7 @@ cron-descriptor==1.4.5
croniter==5.0.1
cryptography==42.0.8
debtcollector==3.0.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@4d500e48e880ea7f9a3582324c55c61373a855ea
Deckhand @ git+https://opendev.org/airship/deckhand.git@211b2d20c080ec806d196f3ec6b0d0472b00fa1b
decorator==5.1.1
deepdiff==8.1.1
Deprecated==1.2.15
@ -63,7 +63,7 @@ dill==0.3.1.1
dnspython==2.7.0
docopt==0.6.2
dogpile.cache==1.3.3
drydock_provisioner @ git+https://opendev.org/airship/drydock.git@ffcd51e00249198f1e2d40f3e83c8c1c338df3ad#subdirectory=python
drydock_provisioner @ git+https://opendev.org/airship/drydock.git@108a3c1ee4666df3a6fb4439df245525a870d1d9#subdirectory=python
email_validator==2.2.0
eventlet==0.38.1
exceptiongroup==1.2.2
@ -188,7 +188,7 @@ pluggy==1.5.0
ply==3.11
prettytable==3.12.0
prison==0.2.1
promenade @ git+https://opendev.org/airship/promenade.git@817ead27e991ff3fbed99ea1cb08aad5674d7167
promenade @ git+https://opendev.org/airship/promenade.git@40fd108fd976c19bc142d10cc52b24d48de37100
prometheus_client==0.21.1
prompt_toolkit==3.0.48
propcache==0.2.1

2
src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml
View File

@ -6,7 +6,7 @@ metadata:
labels:
application: shipyard
data:
$schema: 'http://json-schema.org/schema#'
$schema: 'http://json-schema.org/draft-04/schema#'
id: 'https://github.com/openstack/airship-shipyard/blob/master/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml'
type: 'object'
properties:

2
src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml
View File

@ -6,7 +6,7 @@ metadata:
labels:
application: shipyard
data:
$schema: 'http://json-schema.org/schema#'
$schema: 'http://json-schema.org/draft-04/schema#'
id: 'https://github.com/openstack/airship-shipyard/blob/master/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml'
type: 'object'
required:

150
src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py
View File

@ -12,20 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests ArmadaTestReleasesOperator functionality"""
import os
from unittest import mock
from airflow.exceptions import AirflowException
import pytest
from airflow.exceptions import AirflowException
from shipyard_airflow.plugins.armada_base_operator import \
ArmadaBaseOperator
from shipyard_airflow.plugins.armada_test_releases import \
ArmadaTestReleasesOperator
from shipyard_airflow.plugins.ucp_base_operator import \
UcpBaseOperator
from shipyard_airflow.plugins.armada_base_operator import ArmadaBaseOperator
from shipyard_airflow.plugins.armada_test_releases import ArmadaTestReleasesOperator
from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator
CONF_FILE = os.path.join(os.path.dirname(__file__), 'test.conf')
@ -38,64 +33,81 @@ RELEASES = {
'openstack': ['glance', 'heat', 'horizon', 'keystone']
}
@pytest.fixture
def setup_armada_operator():
"""Fixture to setup the ArmadaTestReleasesOperator with default params"""
op = ArmadaTestReleasesOperator(main_dag_name='main',
shipyard_conf=CONF_FILE,
task_id='t1')
op.action_params = dict()
return op
class TestArmadaTestReleasesOperator(ArmadaTestReleasesOperator):
@mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info')
@mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True)
@mock.patch.object(ArmadaBaseOperator, 'get_releases',
return_value=RELEASES)
def test_do_execute(self, mock_releases, mock_client,
mock_logs):
op = ArmadaTestReleasesOperator(main_dag_name='main',
shipyard_conf=CONF_FILE,
task_id='t1')
op.action_params = dict()
@pytest.fixture
def setup_operator_with_params():
"""Fixture to setup the ArmadaTestReleasesOperator with action params"""
op = ArmadaTestReleasesOperator(main_dag_name='main',
shipyard_conf=CONF_FILE,
task_id='t1')
op.action_params = ACTION_PARAMS
return op
@pytest.fixture
def mock_releases():
"""Fixture to mock releases"""
with mock.patch.object(ArmadaBaseOperator, 'get_releases', return_value=RELEASES):
yield
@pytest.fixture
def mock_client():
"""Fixture to mock Armada client"""
with mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) as mock_client:
yield mock_client
@pytest.fixture
def mock_logs():
"""Fixture to mock logging"""
with mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info') as mock_logs:
yield mock_logs
@pytest.fixture
def mock_k8s_logs():
"""Fixture to mock k8s logs"""
with mock.patch.object(UcpBaseOperator, 'get_k8s_logs') as mock_k8s_logs:
yield mock_k8s_logs
def test_do_execute(setup_armada_operator, mock_releases, mock_client, mock_logs):
"""Test ArmadaTestReleasesOperator execute functionality"""
op = setup_armada_operator
op.do_execute()
# Verify Armada client called to test every release
calls = [mock.call(release=release, timeout=None) for release_list in RELEASES.values() for release in release_list]
mock_client.get_test_release.assert_has_calls(calls, any_order=True)
# Verify test results logged
mock_logs.assert_called_with(mock_client.get_test_release.return_value)
def test_do_execute_with_params(setup_operator_with_params, mock_client, mock_logs):
"""Test ArmadaTestReleasesOperator execute functionality with action params"""
op = setup_operator_with_params
op.do_execute()
# Verify Armada client called for single release with action params
release = ACTION_PARAMS['release']
mock_client.get_test_release.assert_called_once_with(release=release, timeout=None)
# Verify test results logged
mock_logs.assert_called_with(mock_client.get_test_release.return_value)
def test_do_execute_fail(setup_armada_operator, mock_releases, mock_client, mock_k8s_logs):
"""Test failure scenario for ArmadaTestReleasesOperator"""
mock_client.get_test_release.return_value = None
op = setup_armada_operator
# Verify errors logged to pods
with pytest.raises(AirflowException):
op.do_execute()
# Verify Armada client called to test every release
calls = list()
for release_list in RELEASES.values():
for release in release_list:
calls.append(mock.call(
release=release,
timeout=None))
mock_client.get_test_release.assert_has_calls(calls, any_order=True)
# Verify test results logged
mock_logs.assert_called_with(mock_client.get_test_release.return_value)
@mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info')
@mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True)
def test_do_execute_with_params(self, mock_client, mock_logs):
op = ArmadaTestReleasesOperator(main_dag_name='main',
shipyard_conf=CONF_FILE,
task_id='t1')
op.action_params = ACTION_PARAMS
op.do_execute()
# Verify Armada client called for single release with action params
release = ACTION_PARAMS['release']
mock_client.get_test_release.assert_called_once_with(
release=release,
timeout=None)
# Verify test results logged
mock_logs.assert_called_with(mock_client.get_test_release.return_value)
@mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True)
@mock.patch.object(ArmadaBaseOperator, 'get_releases',
return_value=RELEASES)
@mock.patch.object(UcpBaseOperator, 'get_k8s_logs')
def test_do_execute_fail(self, mock_k8s_logs,
mock_releases, mock_client):
mock_client.get_test_release.return_value = None
op = ArmadaTestReleasesOperator(main_dag_name='main',
shipyard_conf=CONF_FILE,
task_id='t1')
op.action_params = dict()
# Verify errors logged to pods
with pytest.raises(AirflowException):
op.do_execute()
mock_k8s_logs.assert_called_once()
mock_k8s_logs.assert_called_once()

5
src/bin/shipyard_airflow/tests/unit/schemas/base_schema_validation.py
View File

@ -16,7 +16,7 @@ import os
import yaml
import jsonschema
import pkg_resources
from importlib.resources import files
import pytest
from jsonschema.exceptions import ValidationError
@ -31,8 +31,7 @@ class BaseSchemaValidationTest(object):
:param expect_failure: should the validation pass or fail.
:param input_files: pytest fixture used to access the test input files
:param input: test input yaml doc filename"""
schema_dir = pkg_resources.resource_filename('shipyard_airflow',
'schemas')
schema_dir = str(files('shipyard_airflow') / 'schemas')
schema_filename = os.path.join(schema_dir, schema)
schema_file = open(schema_filename, 'r')
schema = yaml.safe_load(schema_file)