From 108a3c1ee4666df3a6fb4439df245525a870d1d9 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Mon, 23 Dec 2024 04:16:41 +0000 Subject: [PATCH] Fix deprecated code This PS replaces deprecared module pkg_resources, also fixes the schema validation by adding specific schema draft to choose in order to prevent the processor to fall back to use the latest draft that may potentially cause issues. Also switched to quay.io/airshipit for base ubuntu image Change-Id: I687ef267ee3b027e80815e8852c8edcab5b5b727 --- images/drydock/Dockerfile.ubuntu_jammy | 4 ++-- python/drydock_provisioner/control/bootaction.py | 2 +- .../kubernetes/promenade_driver/promenade_client.py | 2 +- .../drivers/node/maasdriver/models/machine.py | 2 +- python/drydock_provisioner/drydock_client/session.py | 2 +- python/drydock_provisioner/ingester/ingester.py | 2 +- python/drydock_provisioner/ingester/plugins/deckhand.py | 5 ++--- python/drydock_provisioner/ingester/plugins/yaml.py | 5 ++--- python/drydock_provisioner/schemas/baremetalNode.yaml | 2 +- python/drydock_provisioner/schemas/bootaction.yaml | 2 +- python/drydock_provisioner/schemas/hardwareProfile.yaml | 2 +- python/drydock_provisioner/schemas/hostProfile.yaml | 2 +- python/drydock_provisioner/schemas/network.yaml | 2 +- python/drydock_provisioner/schemas/networkLink.yaml | 2 +- python/drydock_provisioner/schemas/rack.yaml | 2 +- python/drydock_provisioner/schemas/region.yaml | 2 +- python/drydock_provisioner/statemgmt/state.py | 2 +- python/test-requirements.txt | 4 ++-- python/tests/unit/test_api_nodes_unit.py | 9 +++++---- python/tests/unit/test_drydock_client_session.py | 4 ---- python/tests/unit/test_schema_validation.py | 5 ++--- 21 files changed, 29 insertions(+), 35 deletions(-) diff --git a/images/drydock/Dockerfile.ubuntu_jammy b/images/drydock/Dockerfile.ubuntu_jammy index 6df22ce7..b9b50f01 100644 --- a/images/drydock/Dockerfile.ubuntu_jammy +++ b/images/drydock/Dockerfile.ubuntu_jammy @@ -18,9 +18,9 @@ # 429 Too Many Requests - Server message: too many requests: # You have reached your pull rate limit. # You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit -ARG FROM=public.ecr.aws/docker/library/ubuntu:jammy +ARG FROM=quay.io/airshipit/ubuntu:jammy -FROM public.ecr.aws/docker/library/golang:1.23.1-bullseye as baclient_builder +FROM quay.io/airshipit/golang:1.23.1-bullseye as baclient_builder COPY ./tools/baclient_build.sh /tmp/drydock/ COPY ./go /tmp/drydock/go diff --git a/python/drydock_provisioner/control/bootaction.py b/python/drydock_provisioner/control/bootaction.py index 656bce2d..6d6c91da 100644 --- a/python/drydock_provisioner/control/bootaction.py +++ b/python/drydock_provisioner/control/bootaction.py @@ -241,7 +241,7 @@ class BootactionUtils(object): challenges=['Bootaction-Key']) if ba_ctx['identity_key'] != bytes.fromhex(identity_key): - logger.warn( + logger.warning( "Forbidding boot action access - node: %s, identity_key: %s, req header: %s" % (ba_ctx['node_name'], str( ba_ctx['identity_key']), str(bytes.fromhex(identity_key)))) diff --git a/python/drydock_provisioner/drivers/kubernetes/promenade_driver/promenade_client.py b/python/drydock_provisioner/drivers/kubernetes/promenade_driver/promenade_client.py index 78110ad3..f4dd6d57 100644 --- a/python/drydock_provisioner/drivers/kubernetes/promenade_driver/promenade_client.py +++ b/python/drydock_provisioner/drivers/kubernetes/promenade_driver/promenade_client.py @@ -203,7 +203,7 @@ class PromenadeSession(object): elif timeout is not None: raise ValueError("Non integer timeout value") except ValueError: - self.logger.warn( + self.logger.warning( "Timeout value must be a tuple of integers or a " "single integer. Proceeding with values of " "(%s, %s)", connect_timeout, read_timeout) diff --git a/python/drydock_provisioner/drivers/node/maasdriver/models/machine.py b/python/drydock_provisioner/drivers/node/maasdriver/models/machine.py index 98aac0fd..42a76b8a 100644 --- a/python/drydock_provisioner/drivers/node/maasdriver/models/machine.py +++ b/python/drydock_provisioner/drivers/node/maasdriver/models/machine.py @@ -631,7 +631,7 @@ class Machines(model_base.ResourceCollectionBase): node_oob_ip = node_model.get_network_address(node_oob_network) if node_oob_ip is None: - self.logger.warn("Node model missing OOB IP address") + self.logger.warning("Node model missing OOB IP address") raise ValueError('Node model missing OOB IP address') maas_node = self.find_node_with_power_address(node_oob_ip) diff --git a/python/drydock_provisioner/drydock_client/session.py b/python/drydock_provisioner/drydock_client/session.py index 0a2dbfd5..81293bef 100644 --- a/python/drydock_provisioner/drydock_client/session.py +++ b/python/drydock_provisioner/drydock_client/session.py @@ -204,7 +204,7 @@ class DrydockSession(object): elif timeout is not None: raise ValueError("Non integer timeout value") except ValueError: - self.logger.warn( + self.logger.warning( "Timeout value must be a tuple of integers or a " "single integer. Proceeding with values of " "(%s, %s)", connect_timeout, read_timeout) diff --git a/python/drydock_provisioner/ingester/ingester.py b/python/drydock_provisioner/ingester/ingester.py index 36c7935e..c9deec2b 100644 --- a/python/drydock_provisioner/ingester/ingester.py +++ b/python/drydock_provisioner/ingester/ingester.py @@ -97,7 +97,7 @@ class Ingester(object): status, design_items = self.registered_plugin.ingest_data( content=design_blob, **kwargs) except errors.IngesterError as vex: - self.logger.warn( + self.logger.warning( "Ingester:ingest_data - Unexpected error processing data - %s" % (str(vex))) return None, None diff --git a/python/drydock_provisioner/ingester/plugins/deckhand.py b/python/drydock_provisioner/ingester/plugins/deckhand.py index 400249d0..31fadae4 100644 --- a/python/drydock_provisioner/ingester/plugins/deckhand.py +++ b/python/drydock_provisioner/ingester/plugins/deckhand.py @@ -17,7 +17,7 @@ import yaml import logging import jsonschema import os -import pkg_resources +from importlib.resources import files import copy import hashlib @@ -717,8 +717,7 @@ class DeckhandIngester(IngesterPlugin): f.close() def _get_schema_dir(self): - return pkg_resources.resource_filename('drydock_provisioner', - 'schemas') + return str(files('drydock_provisioner') / 'schemas') # Mapping of handlers for different document kinds v1_doc_handlers = { diff --git a/python/drydock_provisioner/ingester/plugins/yaml.py b/python/drydock_provisioner/ingester/plugins/yaml.py index 0fb6fba2..1f007467 100644 --- a/python/drydock_provisioner/ingester/plugins/yaml.py +++ b/python/drydock_provisioner/ingester/plugins/yaml.py @@ -18,7 +18,7 @@ import logging import base64 import jsonschema import os -import pkg_resources +from importlib.resources import files import drydock_provisioner.objects.fields as hd_fields @@ -648,8 +648,7 @@ class YamlIngester(IngesterPlugin): f.close() def _get_schema_dir(self): - return pkg_resources.resource_filename('drydock_provisioner', - 'schemas') + return str(files('drydock_provisioner') / 'schemas') # Mapping of handlers for different document kinds v1_doc_handlers = { diff --git a/python/drydock_provisioner/schemas/baremetalNode.yaml b/python/drydock_provisioner/schemas/baremetalNode.yaml index 14779798..8ee2757e 100644 --- a/python/drydock_provisioner/schemas/baremetalNode.yaml +++ b/python/drydock_provisioner/schemas/baremetalNode.yaml @@ -6,7 +6,7 @@ metadata: labels: application: drydock data: - $schema: 'http://json-schema.org/schema#' + $schema: 'http://json-schema.org/draft-04/schema#' id: 'http://att.com/att-comdev/drydock/baremetalNode.yaml' type: 'object' properties: diff --git a/python/drydock_provisioner/schemas/bootaction.yaml b/python/drydock_provisioner/schemas/bootaction.yaml index 504a1df3..0600cdb1 100644 --- a/python/drydock_provisioner/schemas/bootaction.yaml +++ b/python/drydock_provisioner/schemas/bootaction.yaml @@ -6,7 +6,7 @@ metadata: labels: application: drydock data: - $schema: 'http://json-schema.org/schema#' + $schema: 'http://json-schema.org/draft-04/schema#' id: 'http://att.com/att-comdev/drydock/bootaction.yaml' type: 'object' additionalProperties: false diff --git a/python/drydock_provisioner/schemas/hardwareProfile.yaml b/python/drydock_provisioner/schemas/hardwareProfile.yaml index 9d62a7e0..d2c1df5e 100644 --- a/python/drydock_provisioner/schemas/hardwareProfile.yaml +++ b/python/drydock_provisioner/schemas/hardwareProfile.yaml @@ -6,7 +6,7 @@ metadata: labels: application: drydock data: - $schema: 'http://json-schema.org/schema#' + $schema: 'http://json-schema.org/draft-04/schema#' id: 'http://att.com/att-comdev/drydock/hardwareProfile.yaml' type: 'object' properties: diff --git a/python/drydock_provisioner/schemas/hostProfile.yaml b/python/drydock_provisioner/schemas/hostProfile.yaml index 87f74ca8..963187f7 100644 --- a/python/drydock_provisioner/schemas/hostProfile.yaml +++ b/python/drydock_provisioner/schemas/hostProfile.yaml @@ -6,7 +6,7 @@ metadata: labels: application: drydock data: - $schema: 'http://json-schema.org/schema#' + $schema: 'http://json-schema.org/draft-04/schema#' id: 'http://att.com/att-comdev/drydock/hostProfile.yaml' type: 'object' properties: diff --git a/python/drydock_provisioner/schemas/network.yaml b/python/drydock_provisioner/schemas/network.yaml index 4eaaf11d..5033cbc5 100644 --- a/python/drydock_provisioner/schemas/network.yaml +++ b/python/drydock_provisioner/schemas/network.yaml @@ -6,7 +6,7 @@ metadata: labels: application: drydock data: - $schema: 'http://json-schema.org/schema#' + $schema: 'http://json-schema.org/draft-04/schema#' id: 'http://att.com/att-comdev/drydock/network.yaml' type: 'object' properties: diff --git a/python/drydock_provisioner/schemas/networkLink.yaml b/python/drydock_provisioner/schemas/networkLink.yaml index e0f7e34d..819a97a3 100644 --- a/python/drydock_provisioner/schemas/networkLink.yaml +++ b/python/drydock_provisioner/schemas/networkLink.yaml @@ -6,7 +6,7 @@ metadata: labels: application: drydock data: - $schema: 'http://json-schema.org/schema#' + $schema: 'http://json-schema.org/draft-04/schema#' id: 'http://att.com/att-comdev/drydock/networkLink.yaml' type: 'object' properties: diff --git a/python/drydock_provisioner/schemas/rack.yaml b/python/drydock_provisioner/schemas/rack.yaml index 65fe5d13..8187b6d1 100644 --- a/python/drydock_provisioner/schemas/rack.yaml +++ b/python/drydock_provisioner/schemas/rack.yaml @@ -6,7 +6,7 @@ metadata: labels: application: drydock data: - $schema: 'http://json-schema.org/schema#' + $schema: 'http://json-schema.org/draft-04/schema#' id: 'http://att.com/att-comdev/drydock/rack.yaml' type: 'object' properties: diff --git a/python/drydock_provisioner/schemas/region.yaml b/python/drydock_provisioner/schemas/region.yaml index 50d56a19..dc1f6441 100644 --- a/python/drydock_provisioner/schemas/region.yaml +++ b/python/drydock_provisioner/schemas/region.yaml @@ -6,7 +6,7 @@ metadata: labels: application: drydock data: - $schema: 'http://json-schema.org/schema#' + $schema: 'http://json-schema.org/draft-04/schema#' id: 'http://att.com/att-comdev/drydock/region.yaml' type: 'object' properties: diff --git a/python/drydock_provisioner/statemgmt/state.py b/python/drydock_provisioner/statemgmt/state.py index 24abd3bc..12bf0a25 100644 --- a/python/drydock_provisioner/statemgmt/state.py +++ b/python/drydock_provisioner/statemgmt/state.py @@ -49,7 +49,7 @@ class DrydockState(object): max_overflow=config.config_mgr.conf.database.pool_overflow, pool_timeout=config.config_mgr.conf.database.pool_timeout, pool_recycle=config.config_mgr.conf.database.connection_recycle) - self.db_metadata = MetaData(bind=self.db_engine) + self.db_metadata = MetaData() self.tasks_tbl = tables.Tasks(self.db_metadata) self.result_message_tbl = tables.ResultMessage(self.db_metadata) diff --git a/python/test-requirements.txt b/python/test-requirements.txt index 184b107c..a7ab8805 100644 --- a/python/test-requirements.txt +++ b/python/test-requirements.txt @@ -1,5 +1,5 @@ -pytest >= 3.0 -pytest-cov==4.0.0 +pytest +pytest-cov pytest-mock diff --git a/python/tests/unit/test_api_nodes_unit.py b/python/tests/unit/test_api_nodes_unit.py index 438fdada..2d4dc403 100644 --- a/python/tests/unit/test_api_nodes_unit.py +++ b/python/tests/unit/test_api_nodes_unit.py @@ -95,7 +95,8 @@ def mock_process_node_filter(mocker, deckhand_orchestrator): n2.site = 'test2' mock_results = [n1, n2] - with mocker.patch( - 'drydock_provisioner.orchestrator.orchestrator.Orchestrator.process_node_filter', - mocker.MagicMock(return_value=mock_results)): - yield + mocker.patch( + 'drydock_provisioner.orchestrator.orchestrator.Orchestrator.process_node_filter', + return_value=mock_results + ) + yield diff --git a/python/tests/unit/test_drydock_client_session.py b/python/tests/unit/test_drydock_client_session.py index 21fe51a6..348e28c4 100644 --- a/python/tests/unit/test_drydock_client_session.py +++ b/python/tests/unit/test_drydock_client_session.py @@ -53,7 +53,6 @@ class TestClientSession(object): sess = DrydockSession("testdrydock") result = sess.get('bogus') assert result.status_code == 200 - return True @responses.activate def test_get_with_timeout(self): @@ -62,7 +61,6 @@ class TestClientSession(object): sess = DrydockSession("testdrydock") result = sess.get('bogus', timeout=(60, 60)) assert result.status_code == 200 - return True post_responses_inp = { 'method': 'POST', @@ -79,7 +77,6 @@ class TestClientSession(object): sess = DrydockSession("testdrydock") result = sess.post('bogus') assert result.status_code == 200 - return True @responses.activate def test_post_with_timeout(self): @@ -88,7 +85,6 @@ class TestClientSession(object): sess = DrydockSession("testdrydock") result = sess.post('bogus', timeout=(60, 60)) assert result.status_code == 200 - return True def test_timeout(self): """Tests the _timeout method""" diff --git a/python/tests/unit/test_schema_validation.py b/python/tests/unit/test_schema_validation.py index 8a608d21..48e67771 100644 --- a/python/tests/unit/test_schema_validation.py +++ b/python/tests/unit/test_schema_validation.py @@ -1,7 +1,7 @@ import pylibyaml # noqa: F401 # patch pyyaml to use libyaml bindings import yaml import jsonschema -import pkg_resources +from importlib.resources import files import os import shutil import pytest @@ -18,8 +18,7 @@ class BaseSchemaValidationTest(object): :param expect_failure: should the validation pass or fail. :param input_files: pytest fixture used to access the test input files :param input: test input yaml doc filename""" - schema_dir = pkg_resources.resource_filename('drydock_provisioner', - 'schemas') + schema_dir = str(files('drydock_provisioner') / 'schemas') schema_filename = os.path.join(schema_dir, schema) schema_file = open(schema_filename, 'r') schema = yaml.safe_load(schema_file)