diff --git a/images/airflow/Dockerfile.ubuntu_jammy b/images/airflow/Dockerfile.ubuntu_jammy index 62a40cd3..d2257cb0 100644 --- a/images/airflow/Dockerfile.ubuntu_jammy +++ b/images/airflow/Dockerfile.ubuntu_jammy @@ -19,7 +19,7 @@ # 429 Too Many Requests - Server message: too many requests: # You have reached your pull rate limit. # You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit -ARG FROM=public.ecr.aws/docker/library/ubuntu:jammy +ARG FROM=quay.io/airshipit/ubuntu:jammy FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ @@ -55,7 +55,7 @@ ARG DEBIAN_FRONTEND=noninteractive ARG ctx_base=src/bin # Kubectl version -ARG KUBECTL_VERSION=1.31.0 +ARG KUBECTL_VERSION=1.32.0 # Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated ENV AIRFLOW_HOME=${AIRFLOW_HOME} diff --git a/images/shipyard/Dockerfile.ubuntu_jammy b/images/shipyard/Dockerfile.ubuntu_jammy index 99d0e524..fc7b33bd 100644 --- a/images/shipyard/Dockerfile.ubuntu_jammy +++ b/images/shipyard/Dockerfile.ubuntu_jammy @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG FROM=ubuntu:jammy +ARG FROM=quay.io/airshipit/ubuntu:jammy FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ diff --git a/src/bin/shipyard_airflow/requirements-direct.txt b/src/bin/shipyard_airflow/requirements-direct.txt index f792be04..2ae43580 100644 --- a/src/bin/shipyard_airflow/requirements-direct.txt +++ b/src/bin/shipyard_airflow/requirements-direct.txt @@ -59,7 +59,7 @@ oslo.versionedobjects==3.3.0 # Airship dependencies -git+https://opendev.org/airship/deckhand.git@4d500e48e880ea7f9a3582324c55c61373a855ea#egg=deckhand -git+https://opendev.org/airship/drydock.git@ffcd51e00249198f1e2d40f3e83c8c1c338df3ad#egg=drydock_provisioner&subdirectory=python -git+https://opendev.org/airship/armada.git@af4bf814f3a44e9ccee7c9fd8e4be34d943e013d#egg=armada -git+https://opendev.org/airship/promenade.git@817ead27e991ff3fbed99ea1cb08aad5674d7167#egg=promenade +git+https://opendev.org/airship/deckhand.git@211b2d20c080ec806d196f3ec6b0d0472b00fa1b#egg=deckhand +git+https://opendev.org/airship/drydock.git@108a3c1ee4666df3a6fb4439df245525a870d1d9#egg=drydock_provisioner&subdirectory=python +git+https://opendev.org/airship/armada.git@963e23bb9323ea30472c54adc32e5c9eb7bf0506#egg=armada +git+https://opendev.org/airship/promenade.git@40fd108fd976c19bc142d10cc52b24d48de37100#egg=promenade diff --git a/src/bin/shipyard_airflow/requirements-frozen.txt b/src/bin/shipyard_airflow/requirements-frozen.txt index 8ec69613..8eb87c04 100644 --- a/src/bin/shipyard_airflow/requirements-frozen.txt +++ b/src/bin/shipyard_airflow/requirements-frozen.txt @@ -21,7 +21,7 @@ apache-airflow-providers-smtp==1.8.1 apache-airflow-providers-sqlite==3.9.1 apispec==6.8.0 argcomplete==3.5.2 -Armada @ git+https://opendev.org/airship/armada.git@af4bf814f3a44e9ccee7c9fd8e4be34d943e013d +Armada @ git+https://opendev.org/airship/armada.git@963e23bb9323ea30472c54adc32e5c9eb7bf0506 arrow==1.3.0 asgiref==3.8.1 async-timeout==5.0.1 @@ -55,7 +55,7 @@ cron-descriptor==1.4.5 croniter==5.0.1 cryptography==42.0.8 debtcollector==3.0.0 -Deckhand @ git+https://opendev.org/airship/deckhand.git@4d500e48e880ea7f9a3582324c55c61373a855ea +Deckhand @ git+https://opendev.org/airship/deckhand.git@211b2d20c080ec806d196f3ec6b0d0472b00fa1b decorator==5.1.1 deepdiff==8.1.1 Deprecated==1.2.15 @@ -63,7 +63,7 @@ dill==0.3.1.1 dnspython==2.7.0 docopt==0.6.2 dogpile.cache==1.3.3 -drydock_provisioner @ git+https://opendev.org/airship/drydock.git@ffcd51e00249198f1e2d40f3e83c8c1c338df3ad#subdirectory=python +drydock_provisioner @ git+https://opendev.org/airship/drydock.git@108a3c1ee4666df3a6fb4439df245525a870d1d9#subdirectory=python email_validator==2.2.0 eventlet==0.38.1 exceptiongroup==1.2.2 @@ -188,7 +188,7 @@ pluggy==1.5.0 ply==3.11 prettytable==3.12.0 prison==0.2.1 -promenade @ git+https://opendev.org/airship/promenade.git@817ead27e991ff3fbed99ea1cb08aad5674d7167 +promenade @ git+https://opendev.org/airship/promenade.git@40fd108fd976c19bc142d10cc52b24d48de37100 prometheus_client==0.21.1 prompt_toolkit==3.0.48 propcache==0.2.1 diff --git a/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml b/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml index d79d313a..d6c94180 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml +++ b/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml @@ -6,7 +6,7 @@ metadata: labels: application: shipyard data: - $schema: 'http://json-schema.org/schema#' + $schema: 'http://json-schema.org/draft-04/schema#' id: 'https://github.com/openstack/airship-shipyard/blob/master/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentConfiguration.yaml' type: 'object' properties: diff --git a/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml b/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml index c51e40cd..234dbd00 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml +++ b/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml @@ -6,7 +6,7 @@ metadata: labels: application: shipyard data: - $schema: 'http://json-schema.org/schema#' + $schema: 'http://json-schema.org/draft-04/schema#' id: 'https://github.com/openstack/airship-shipyard/blob/master/src/bin/shipyard_airflow/shipyard_airflow/schemas/deploymentStrategy.yaml' type: 'object' required: diff --git a/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py b/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py index e965af39..6dc5a336 100644 --- a/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py +++ b/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py @@ -12,20 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. """Tests ArmadaTestReleasesOperator functionality""" + import os from unittest import mock - -from airflow.exceptions import AirflowException import pytest +from airflow.exceptions import AirflowException -from shipyard_airflow.plugins.armada_base_operator import \ - ArmadaBaseOperator -from shipyard_airflow.plugins.armada_test_releases import \ - ArmadaTestReleasesOperator -from shipyard_airflow.plugins.ucp_base_operator import \ - UcpBaseOperator - - +from shipyard_airflow.plugins.armada_base_operator import ArmadaBaseOperator +from shipyard_airflow.plugins.armada_test_releases import ArmadaTestReleasesOperator +from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator CONF_FILE = os.path.join(os.path.dirname(__file__), 'test.conf') @@ -38,64 +33,81 @@ RELEASES = { 'openstack': ['glance', 'heat', 'horizon', 'keystone'] } +@pytest.fixture +def setup_armada_operator(): + """Fixture to setup the ArmadaTestReleasesOperator with default params""" + op = ArmadaTestReleasesOperator(main_dag_name='main', + shipyard_conf=CONF_FILE, + task_id='t1') + op.action_params = dict() + return op -class TestArmadaTestReleasesOperator(ArmadaTestReleasesOperator): - @mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info') - @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) - @mock.patch.object(ArmadaBaseOperator, 'get_releases', - return_value=RELEASES) - def test_do_execute(self, mock_releases, mock_client, - mock_logs): - op = ArmadaTestReleasesOperator(main_dag_name='main', - shipyard_conf=CONF_FILE, - task_id='t1') - op.action_params = dict() +@pytest.fixture +def setup_operator_with_params(): + """Fixture to setup the ArmadaTestReleasesOperator with action params""" + op = ArmadaTestReleasesOperator(main_dag_name='main', + shipyard_conf=CONF_FILE, + task_id='t1') + op.action_params = ACTION_PARAMS + return op + +@pytest.fixture +def mock_releases(): + """Fixture to mock releases""" + with mock.patch.object(ArmadaBaseOperator, 'get_releases', return_value=RELEASES): + yield + +@pytest.fixture +def mock_client(): + """Fixture to mock Armada client""" + with mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) as mock_client: + yield mock_client + +@pytest.fixture +def mock_logs(): + """Fixture to mock logging""" + with mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info') as mock_logs: + yield mock_logs + +@pytest.fixture +def mock_k8s_logs(): + """Fixture to mock k8s logs""" + with mock.patch.object(UcpBaseOperator, 'get_k8s_logs') as mock_k8s_logs: + yield mock_k8s_logs + +def test_do_execute(setup_armada_operator, mock_releases, mock_client, mock_logs): + """Test ArmadaTestReleasesOperator execute functionality""" + op = setup_armada_operator + op.do_execute() + + # Verify Armada client called to test every release + calls = [mock.call(release=release, timeout=None) for release_list in RELEASES.values() for release in release_list] + mock_client.get_test_release.assert_has_calls(calls, any_order=True) + + # Verify test results logged + mock_logs.assert_called_with(mock_client.get_test_release.return_value) + + +def test_do_execute_with_params(setup_operator_with_params, mock_client, mock_logs): + """Test ArmadaTestReleasesOperator execute functionality with action params""" + op = setup_operator_with_params + op.do_execute() + + # Verify Armada client called for single release with action params + release = ACTION_PARAMS['release'] + mock_client.get_test_release.assert_called_once_with(release=release, timeout=None) + + # Verify test results logged + mock_logs.assert_called_with(mock_client.get_test_release.return_value) + + +def test_do_execute_fail(setup_armada_operator, mock_releases, mock_client, mock_k8s_logs): + """Test failure scenario for ArmadaTestReleasesOperator""" + mock_client.get_test_release.return_value = None + + op = setup_armada_operator + + # Verify errors logged to pods + with pytest.raises(AirflowException): op.do_execute() - - # Verify Armada client called to test every release - calls = list() - for release_list in RELEASES.values(): - for release in release_list: - calls.append(mock.call( - release=release, - timeout=None)) - mock_client.get_test_release.assert_has_calls(calls, any_order=True) - - # Verify test results logged - mock_logs.assert_called_with(mock_client.get_test_release.return_value) - - @mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info') - @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) - def test_do_execute_with_params(self, mock_client, mock_logs): - op = ArmadaTestReleasesOperator(main_dag_name='main', - shipyard_conf=CONF_FILE, - task_id='t1') - op.action_params = ACTION_PARAMS - op.do_execute() - - # Verify Armada client called for single release with action params - release = ACTION_PARAMS['release'] - mock_client.get_test_release.assert_called_once_with( - release=release, - timeout=None) - - # Verify test results logged - mock_logs.assert_called_with(mock_client.get_test_release.return_value) - - @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) - @mock.patch.object(ArmadaBaseOperator, 'get_releases', - return_value=RELEASES) - @mock.patch.object(UcpBaseOperator, 'get_k8s_logs') - def test_do_execute_fail(self, mock_k8s_logs, - mock_releases, mock_client): - mock_client.get_test_release.return_value = None - - op = ArmadaTestReleasesOperator(main_dag_name='main', - shipyard_conf=CONF_FILE, - task_id='t1') - op.action_params = dict() - - # Verify errors logged to pods - with pytest.raises(AirflowException): - op.do_execute() - mock_k8s_logs.assert_called_once() + mock_k8s_logs.assert_called_once() diff --git a/src/bin/shipyard_airflow/tests/unit/schemas/base_schema_validation.py b/src/bin/shipyard_airflow/tests/unit/schemas/base_schema_validation.py index 13685769..2d68967b 100644 --- a/src/bin/shipyard_airflow/tests/unit/schemas/base_schema_validation.py +++ b/src/bin/shipyard_airflow/tests/unit/schemas/base_schema_validation.py @@ -16,7 +16,7 @@ import os import yaml import jsonschema -import pkg_resources +from importlib.resources import files import pytest from jsonschema.exceptions import ValidationError @@ -31,8 +31,7 @@ class BaseSchemaValidationTest(object): :param expect_failure: should the validation pass or fail. :param input_files: pytest fixture used to access the test input files :param input: test input yaml doc filename""" - schema_dir = pkg_resources.resource_filename('shipyard_airflow', - 'schemas') + schema_dir = str(files('shipyard_airflow') / 'schemas') schema_filename = os.path.join(schema_dir, schema) schema_file = open(schema_filename, 'r') schema = yaml.safe_load(schema_file)