From 86b7d034032e4399c5fc4a901c59616ebeaaae62 Mon Sep 17 00:00:00 2001
From: Sergiy Markin <smarkin@mirantis.com>
Date: Mon, 23 Dec 2024 06:03:19 +0000
Subject: [PATCH] Fix deprecated code
This PS replaces deprecared module pkg_resources,
also fixes some unit tests and fixes the schema
validation by adding specific schema draft to
choose in order to prevent the processor to fall
back to use the latest draft that may potentially
cause issues.
Also this PS bumps kubectl version to 1.32.0 due
to CVE and switched base ubuntu image repo to
quay.io/airshipit
Change-Id: Ie40f179eac83fde4d828e6f63a9c03e473eb3b15
---
images/airflow/Dockerfile.ubuntu_jammy | 4 +-
images/shipyard/Dockerfile.ubuntu_jammy | 2 +-
.../shipyard_airflow/requirements-direct.txt | 8 +-
.../shipyard_airflow/requirements-frozen.txt | 8 +-
.../schemas/deploymentConfiguration.yaml | 2 +-
.../schemas/deploymentStrategy.yaml | 2 +-
.../test_armada_test_releases_operator.py | 150 ++++++++++--------
.../unit/schemas/base_schema_validation.py | 5 +-
8 files changed, 96 insertions(+), 85 deletions(-)
diff --git a/images/airflow/Dockerfile.ubuntu_jammy b/images/airflow/Dockerfile.ubuntu_jammy
index 62a40cd3..d2257cb0 100644
--- a/images/airflow/Dockerfile.ubuntu_jammy
+++ b/images/airflow/Dockerfile.ubuntu_jammy
@@ -19,7 +19,7 @@
# 429 Too Many Requests - Server message: too many requests:
# You have reached your pull rate limit.
# You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit
-ARG FROM=public.ecr.aws/docker/library/ubuntu:jammy
+ARG FROM=quay.io/airshipit/ubuntu:jammy
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
@@ -55,7 +55,7 @@ ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin
# Kubectl version
-ARG KUBECTL_VERSION=1.31.0
+ARG KUBECTL_VERSION=1.32.0
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
diff --git a/images/shipyard/Dockerfile.ubuntu_jammy b/images/shipyard/Dockerfile.ubuntu_jammy
index 99d0e524..fc7b33bd 100644
--- a/images/shipyard/Dockerfile.ubuntu_jammy
+++ b/images/shipyard/Dockerfile.ubuntu_jammy
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-ARG FROM=ubuntu:jammy
+ARG FROM=quay.io/airshipit/ubuntu:jammy
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
diff --git a/src/bin/shipyard_airflow/requirements-direct.txt b/src/bin/shipyard_airflow/requirements-direct.txt
index f792be04..2ae43580 100644
--- a/src/bin/shipyard_airflow/requirements-direct.txt
+++ b/src/bin/shipyard_airflow/requirements-direct.txt
@@ -59,7 +59,7 @@ oslo.versionedobjects==3.3.0
# Airship dependencies
-git+https://opendev.org/airship/deckhand.git@4d500e48e880ea7f9a3582324c55c61373a855ea#egg=deckhand
-git+https://opendev.org/airship/drydock.git@ffcd51e00249198f1e2d40f3e83c8c1c338df3ad#egg=drydock_provisioner&subdirectory=python
-git+https://opendev.org/airship/armada.git@af4bf814f3a44e9ccee7c9fd8e4be34d943e013d#egg=armada
-git+https://opendev.org/airship/promenade.git@817ead27e991ff3fbed99ea1cb08aad5674d7167#egg=promenade
+git+https://opendev.org/airship/deckhand.git@211b2d20c080ec806d196f3ec6b0d0472b00fa1b#egg=deckhand
+git+https://opendev.org/airship/drydock.git@108a3c1ee4666df3a6fb4439df245525a870d1d9#egg=drydock_provisioner&subdirectory=python
+git+https://opendev.org/airship/armada.git@963e23bb9323ea30472c54adc32e5c9eb7bf0506#egg=armada
+git+https://opendev.org/airship/promenade.git@40fd108fd976c19bc142d10cc52b24d48de37100#egg=promenade
diff --git a/src/bin/shipyard_airflow/requirements-frozen.txt b/src/bin/shipyard_airflow/requirements-frozen.txt
index 8ec69613..8eb87c04 100644
--- a/src/bin/shipyard_airflow/requirements-frozen.txt
+++ b/src/bin/shipyard_airflow/requirements-frozen.txt
@@ -21,7 +21,7 @@ apache-airflow-providers-smtp==1.8.1
apache-airflow-providers-sqlite==3.9.1
apispec==6.8.0
argcomplete==3.5.2
-Armada @ git+https://opendev.org/airship/armada.git@af4bf814f3a44e9ccee7c9fd8e4be34d943e013d
+Armada @ git+https://opendev.org/airship/armada.git@963e23bb9323ea30472c54adc32e5c9eb7bf0506
arrow==1.3.0
asgiref==3.8.1
async-timeout==5.0.1
@@ -55,7 +55,7 @@ cron-descriptor==1.4.5
croniter==5.0.1
cryptography==42.0.8
debtcollector==3.0.0
-Deckhand @ git+https://opendev.org/airship/deckhand.git@4d500e48e880ea7f9a3582324c55c61373a855ea
+Deckhand @ git+https://opendev.org/airship/deckhand.git@211b2d20c080ec806d196f3ec6b0d0472b00fa1b
decorator==5.1.1
deepdiff==8.1.1
Deprecated==1.2.15
@@ -63,7 +63,7 @@ dill==0.3.1.1
dnspython==2.7.0
docopt==0.6.2
dogpile.cache==1.3.3
-drydock_provisioner @ git+https://opendev.org/airship/drydock.git@ffcd51e00249198f1e2d40f3e83c8c1c338df3ad#subdirectory=python
+drydock_provisioner @ git+https://opendev.org/airship/drydock.git@108a3c1ee4666df3a6fb4439df245525a870d1d9#subdirectory=python
email_validator==2.2.0
eventlet==0.38.1
exceptiongroup==1.2.2
@@ -188,7 +188,7 @@ pluggy==1.5.0
ply==3.11
prettytable==3.12.0
prison==0.2.1
-promenade @ git+https://opendev.org/airship/promenade.git@817ead27e991ff3fbed99ea1cb08aad5674d7167
+promenade @ git+https://opendev.org/airship/promenade.git@40fd108fd976c19bc142d10cc52b24d48de37100
prometheus_client==0.21.1
prompt_toolkit==3.0.48
propcache==0.2.1
diff --git a/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml b/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml
index d79d313a..d6c94180 100644
--- a/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml
+++ b/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml
@@ -6,7 +6,7 @@ metadata:
labels:
application: shipyard
data:
- $schema: 'http://json-schema.org/schema#'
+ $schema: 'http://json-schema.org/draft-04/schema#'
id: 'https://github.com/openstack/airship-shipyard/blob/master/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml'
type: 'object'
properties:
diff --git a/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml b/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml
index c51e40cd..234dbd00 100644
--- a/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml
+++ b/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml
@@ -6,7 +6,7 @@ metadata:
labels:
application: shipyard
data:
- $schema: 'http://json-schema.org/schema#'
+ $schema: 'http://json-schema.org/draft-04/schema#'
id: 'https://github.com/openstack/airship-shipyard/blob/master/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml'
type: 'object'
required:
diff --git a/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py b/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py
index e965af39..6dc5a336 100644
--- a/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py
+++ b/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py
@@ -12,20 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests ArmadaTestReleasesOperator functionality"""
+
import os
from unittest import mock
-
-from airflow.exceptions import AirflowException
import pytest
+from airflow.exceptions import AirflowException
-from shipyard_airflow.plugins.armada_base_operator import \
- ArmadaBaseOperator
-from shipyard_airflow.plugins.armada_test_releases import \
- ArmadaTestReleasesOperator
-from shipyard_airflow.plugins.ucp_base_operator import \
- UcpBaseOperator
-
-
+from shipyard_airflow.plugins.armada_base_operator import ArmadaBaseOperator
+from shipyard_airflow.plugins.armada_test_releases import ArmadaTestReleasesOperator
+from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator
CONF_FILE = os.path.join(os.path.dirname(__file__), 'test.conf')
@@ -38,64 +33,81 @@ RELEASES = {
'openstack': ['glance', 'heat', 'horizon', 'keystone']
}
+@pytest.fixture
+def setup_armada_operator():
+ """Fixture to setup the ArmadaTestReleasesOperator with default params"""
+ op = ArmadaTestReleasesOperator(main_dag_name='main',
+ shipyard_conf=CONF_FILE,
+ task_id='t1')
+ op.action_params = dict()
+ return op
-class TestArmadaTestReleasesOperator(ArmadaTestReleasesOperator):
- @mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info')
- @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True)
- @mock.patch.object(ArmadaBaseOperator, 'get_releases',
- return_value=RELEASES)
- def test_do_execute(self, mock_releases, mock_client,
- mock_logs):
- op = ArmadaTestReleasesOperator(main_dag_name='main',
- shipyard_conf=CONF_FILE,
- task_id='t1')
- op.action_params = dict()
+@pytest.fixture
+def setup_operator_with_params():
+ """Fixture to setup the ArmadaTestReleasesOperator with action params"""
+ op = ArmadaTestReleasesOperator(main_dag_name='main',
+ shipyard_conf=CONF_FILE,
+ task_id='t1')
+ op.action_params = ACTION_PARAMS
+ return op
+
+@pytest.fixture
+def mock_releases():
+ """Fixture to mock releases"""
+ with mock.patch.object(ArmadaBaseOperator, 'get_releases', return_value=RELEASES):
+ yield
+
+@pytest.fixture
+def mock_client():
+ """Fixture to mock Armada client"""
+ with mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) as mock_client:
+ yield mock_client
+
+@pytest.fixture
+def mock_logs():
+ """Fixture to mock logging"""
+ with mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info') as mock_logs:
+ yield mock_logs
+
+@pytest.fixture
+def mock_k8s_logs():
+ """Fixture to mock k8s logs"""
+ with mock.patch.object(UcpBaseOperator, 'get_k8s_logs') as mock_k8s_logs:
+ yield mock_k8s_logs
+
+def test_do_execute(setup_armada_operator, mock_releases, mock_client, mock_logs):
+ """Test ArmadaTestReleasesOperator execute functionality"""
+ op = setup_armada_operator
+ op.do_execute()
+
+ # Verify Armada client called to test every release
+ calls = [mock.call(release=release, timeout=None) for release_list in RELEASES.values() for release in release_list]
+ mock_client.get_test_release.assert_has_calls(calls, any_order=True)
+
+ # Verify test results logged
+ mock_logs.assert_called_with(mock_client.get_test_release.return_value)
+
+
+def test_do_execute_with_params(setup_operator_with_params, mock_client, mock_logs):
+ """Test ArmadaTestReleasesOperator execute functionality with action params"""
+ op = setup_operator_with_params
+ op.do_execute()
+
+ # Verify Armada client called for single release with action params
+ release = ACTION_PARAMS['release']
+ mock_client.get_test_release.assert_called_once_with(release=release, timeout=None)
+
+ # Verify test results logged
+ mock_logs.assert_called_with(mock_client.get_test_release.return_value)
+
+
+def test_do_execute_fail(setup_armada_operator, mock_releases, mock_client, mock_k8s_logs):
+ """Test failure scenario for ArmadaTestReleasesOperator"""
+ mock_client.get_test_release.return_value = None
+
+ op = setup_armada_operator
+
+ # Verify errors logged to pods
+ with pytest.raises(AirflowException):
op.do_execute()
-
- # Verify Armada client called to test every release
- calls = list()
- for release_list in RELEASES.values():
- for release in release_list:
- calls.append(mock.call(
- release=release,
- timeout=None))
- mock_client.get_test_release.assert_has_calls(calls, any_order=True)
-
- # Verify test results logged
- mock_logs.assert_called_with(mock_client.get_test_release.return_value)
-
- @mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info')
- @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True)
- def test_do_execute_with_params(self, mock_client, mock_logs):
- op = ArmadaTestReleasesOperator(main_dag_name='main',
- shipyard_conf=CONF_FILE,
- task_id='t1')
- op.action_params = ACTION_PARAMS
- op.do_execute()
-
- # Verify Armada client called for single release with action params
- release = ACTION_PARAMS['release']
- mock_client.get_test_release.assert_called_once_with(
- release=release,
- timeout=None)
-
- # Verify test results logged
- mock_logs.assert_called_with(mock_client.get_test_release.return_value)
-
- @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True)
- @mock.patch.object(ArmadaBaseOperator, 'get_releases',
- return_value=RELEASES)
- @mock.patch.object(UcpBaseOperator, 'get_k8s_logs')
- def test_do_execute_fail(self, mock_k8s_logs,
- mock_releases, mock_client):
- mock_client.get_test_release.return_value = None
-
- op = ArmadaTestReleasesOperator(main_dag_name='main',
- shipyard_conf=CONF_FILE,
- task_id='t1')
- op.action_params = dict()
-
- # Verify errors logged to pods
- with pytest.raises(AirflowException):
- op.do_execute()
- mock_k8s_logs.assert_called_once()
+ mock_k8s_logs.assert_called_once()
diff --git a/src/bin/shipyard_airflow/tests/unit/schemas/base_schema_validation.py b/src/bin/shipyard_airflow/tests/unit/schemas/base_schema_validation.py
index 13685769..2d68967b 100644
--- a/src/bin/shipyard_airflow/tests/unit/schemas/base_schema_validation.py
+++ b/src/bin/shipyard_airflow/tests/unit/schemas/base_schema_validation.py
@@ -16,7 +16,7 @@ import os
import yaml
import jsonschema
-import pkg_resources
+from importlib.resources import files
import pytest
from jsonschema.exceptions import ValidationError
@@ -31,8 +31,7 @@ class BaseSchemaValidationTest(object):
:param expect_failure: should the validation pass or fail.
:param input_files: pytest fixture used to access the test input files
:param input: test input yaml doc filename"""
- schema_dir = pkg_resources.resource_filename('shipyard_airflow',
- 'schemas')
+ schema_dir = str(files('shipyard_airflow') / 'schemas')
schema_filename = os.path.join(schema_dir, schema)
schema_file = open(schema_filename, 'r')
schema = yaml.safe_load(schema_file)