From eeb1976e632f7e5a7600142d369ec69bdc76ff95 Mon Sep 17 00:00:00 2001 From: stewie925 Date: Fri, 13 Sep 2019 10:52:38 -0700 Subject: [PATCH] Ranger - remove git logic Update Ranger to remove logic to stage heat template to git repo. Instead, pass heat template directly to ranger-agent. Added logic to create/update/delete heat template entry in rds database. Remove git repo logic from ranger. Ranger to pass token to ranger-agent for request authentication. Reintroduce commit #693590 changes that were inadvertently removed. Change-Id: If95076e8343bbb50a7231b74fd4a80906b2a0785 --- etc/ranger/ranger.conf | 2 - images/ranger/Dockerfile | 1 - .../client/keystone/keystone_utils/tokens.py | 3 +- orm/services/resource_distributor/config.py | 19 --- orm/services/resource_distributor/rds/app.py | 22 --- .../controllers/v1/status/resource_status.py | 2 +- .../rds/ordupdate/ord_notifier.py | 156 ++++++++++++------ .../model/region_resource_id_status.py | 17 +- .../rds/services/region_resource_id_status.py | 60 ++++++- .../rds/services/resource.py | 85 ++++++++-- .../resource_distributor/rds/sot/__init__.py | 0 .../rds/sot/git_sot/__init__.py | 0 .../rds/sot/sot_factory.py | 29 ---- .../rds/storage/factory.py | 14 +- .../mysql/region_resource_id_status.py | 150 ++++++++++++++--- .../rds/storage/region_resource_id_status.py | 25 ++- .../resource_distributor/rds/utils/utils.py | 13 +- .../scripts/db_scripts/create_db.sql | 40 +++-- .../v1/status/test_get_resource_status.py | 4 +- .../unit/rds/ordupdate/test_ord_notifier.py | 93 ++++++++--- .../model/test_region_resource_id_status.py | 31 +++- .../unit/rds/services/test_create_resource.py | 29 +--- orm/tests/unit/rds/sot/__init__.py | 0 orm/tests/unit/rds/sot/git_sot/__init__.py | 0 orm/tests/unit/rds/sot/test_base_sot.py | 18 -- .../mysql/test_region_resource_id_status.py | 44 ++--- .../storage/test_region_resource_id_status.py | 2 - requirements.txt | 4 - 28 files changed, 563 insertions(+), 300 deletions(-) delete mode 100644 orm/services/resource_distributor/rds/sot/__init__.py delete mode 100644 orm/services/resource_distributor/rds/sot/git_sot/__init__.py delete mode 100644 orm/services/resource_distributor/rds/sot/sot_factory.py delete mode 100644 orm/tests/unit/rds/sot/__init__.py delete mode 100644 orm/tests/unit/rds/sot/git_sot/__init__.py delete mode 100644 orm/tests/unit/rds/sot/test_base_sot.py diff --git a/etc/ranger/ranger.conf b/etc/ranger/ranger.conf index d37b4b29..4c66760b 100644 --- a/etc/ranger/ranger.conf +++ b/etc/ranger/ranger.conf @@ -65,8 +65,6 @@ log = 'rms.log' [rds] port = 8777 -repo_local_location = '/opt/stack/ranger/orm_repo' -repo_remote_location = 'git@127.0.0.1:/opt/stack/ranger/orm_repo.git' log = 'rdstest.log' [cli] diff --git a/images/ranger/Dockerfile b/images/ranger/Dockerfile index 09bc6fc0..cbbf0601 100644 --- a/images/ranger/Dockerfile +++ b/images/ranger/Dockerfile @@ -51,7 +51,6 @@ RUN useradd -u 1000 -ms /bin/false ${user} # Change permissions RUN mkdir -p /etc/ranger \ && mkdir /var/log/ranger \ - && mkdir /home/${user}/git_repo \ && chown -R ${user}: /var/log/ranger \ && mv /tmp/ranger /home/${user}/ranger \ && chown -R ${user}: /home/${user} \ diff --git a/orm/common/client/keystone/keystone_utils/tokens.py b/orm/common/client/keystone/keystone_utils/tokens.py index cb5fb28a..66618951 100755 --- a/orm/common/client/keystone/keystone_utils/tokens.py +++ b/orm/common/client/keystone/keystone_utils/tokens.py @@ -82,6 +82,7 @@ def get_token_user(token, conf, lcp_id=None, keystone_ep=None): logger.debug(message) raise ValueError(message) keystone_ep = _find_keystone_ep(conf.rms_url, lcp_id, token) + if keystone_ep is None: message = 'Keystone EP of LCP %s not found in RMS' % (lcp_id,) logger.debug(message) @@ -152,7 +153,7 @@ def _find_keystone_ep(rms_url, lcp_name, token): return endpoint['publicURL'] except KeyError: logger.debug('Response from RMS came in an unsupported format. ' - 'Make sure that you are using RMS 3.5') + 'Make sure that you are using the correct format.') return None # Keystone EP not found in the response diff --git a/orm/services/resource_distributor/config.py b/orm/services/resource_distributor/config.py index 1376e41f..8c527b4e 100755 --- a/orm/services/resource_distributor/config.py +++ b/orm/services/resource_distributor/config.py @@ -18,25 +18,6 @@ database = { 'connection_string': config.db_connect } -sot = { - 'type': 'git', -} - -git = { - # possible values : 'native', 'gittle' - 'type': 'native', - 'local_repository_path': config.rds['repo_local_location'], - 'file_name_format': 's_{}.yml', - 'relative_path_format': '/{}/hot/{}/{}', - 'commit_message_format': 'File was added to repository: {}', - 'commit_user': config.rds['repo_user'], - 'commit_email': config.rds['repo_email'], - 'git_server_url': config.rds['repo_remote_location'], - 'git_cmd_timeout': 45, - 'git_retries': config.rds['git_retries'], - 'git_retries_interval': config.rds['git_retries_interval'] -} - audit = { 'audit_server_url': config.audit_server['base_url'] + 'v1/audit/transaction', 'num_of_send_retries': 3, diff --git a/orm/services/resource_distributor/rds/app.py b/orm/services/resource_distributor/rds/app.py index 776518bf..b0d7827d 100755 --- a/orm/services/resource_distributor/rds/app.py +++ b/orm/services/resource_distributor/rds/app.py @@ -3,7 +3,6 @@ import os import sys from .services import region_resource_id_status -from .sot import sot_factory from .storage import factory from oslo_config import cfg @@ -25,7 +24,6 @@ def setup_app(pecan_config): The method initializes components and return a WSGI application """ - init_sot() init_audit() factory.database = conf.database @@ -34,25 +32,9 @@ def setup_app(pecan_config): app = make_app(conf.app.root, logging=conf.logging) logger.info('Starting RDS...') - validate_sot() - return app -def init_sot(): - """Initialize SoT module - """ - sot_factory.sot_type = conf.sot.type - sot_factory.local_repository_path = conf.git.local_repository_path - sot_factory.relative_path_format = conf.git.relative_path_format - sot_factory.file_name_format = conf.git.file_name_format - sot_factory.commit_message_format = conf.git.commit_message_format - sot_factory.commit_user = conf.git.commit_user - sot_factory.commit_email = conf.git.commit_email - sot_factory.git_server_url = conf.git.git_server_url - sot_factory.git_type = conf.git.type - - def init_audit(): """Initialize audit client module """ @@ -62,10 +44,6 @@ def init_audit(): conf.app.service_name) -def validate_sot(): - sot_factory.get_sot().validate_sot_state() - - def main(argv=None): if argv is None: argv = sys.argv diff --git a/orm/services/resource_distributor/rds/controllers/v1/status/resource_status.py b/orm/services/resource_distributor/rds/controllers/v1/status/resource_status.py index 3c98a574..6828a3ce 100755 --- a/orm/services/resource_distributor/rds/controllers/v1/status/resource_status.py +++ b/orm/services/resource_distributor/rds/controllers/v1/status/resource_status.py @@ -145,7 +145,7 @@ class Status(rest.RestController): logger.debug("save data to database.. data :- %s" % data_to_save) try: regionResourceIdStatus.add_status(data_to_save) - # invoke regin data to delete on sucess + # invoke region data to delete on sucess utils.invoke_delete_region(data_to_save) # send data to ims utils.post_data_to_image(data_to_save) diff --git a/orm/services/resource_distributor/rds/ordupdate/ord_notifier.py b/orm/services/resource_distributor/rds/ordupdate/ord_notifier.py index 79019721..a04f42da 100755 --- a/orm/services/resource_distributor/rds/ordupdate/ord_notifier.py +++ b/orm/services/resource_distributor/rds/ordupdate/ord_notifier.py @@ -1,5 +1,6 @@ """ORD trigger main module.""" +import base64 import json import logging import requests @@ -7,6 +8,8 @@ import time from orm.common.client.audit.audit_client.api import audit from orm.services.resource_distributor.rds.services import region_resource_id_status as regionResourceIdStatus +from orm.services.resource_distributor.rds.utils import \ + authentication as AuthService from pecan import conf @@ -17,6 +20,12 @@ ACK_CODE = 200 logger = logging.getLogger(__name__) +class NoTokenError(Exception): + """Indicates that no token is received.""" + + pass + + class OrdNotFoundError(Exception): """Indicates that the correct ORD to notify was not found.""" @@ -70,7 +79,8 @@ def _notify(ord_url, resource_template_version, resource_template_name, operation, - region_id): + region_id, + resource_template_data): """Send the notification message to the ORD. :param ord_url: @@ -81,13 +91,25 @@ def _notify(ord_url, :param resource_template_name: :param operation: :param region_id: + :param resource_template_data: :raise: requests.exceptions.ConnectionError when the POST request cannot be sent, - NotifyNotAcknowledgedError when the ORD did not respond to the notification - as expected + NotifyNotAcknowledgedError when ranger agent did not respond to the + notification as expected InvalidJsonError if the payload is missing one of the expected values :return: """ + # get token for region and send to ranger-agent if authenticate enabled + headers = {} + if conf.authentication.enabled: + token_id = AuthService.get_token(region_id) + + if token_id: + headers['X-Auth-Token'] = token_id + else: + logger.info('Failed get_token for region %s' % (region_id)) + raise NoTokenError + # Prepare the request body data_to_send = { 'ord-notifier': { @@ -106,8 +128,13 @@ def _notify(ord_url, https_enabled = conf.ordupdate.https_enabled logger.debug('notify: ord_url: %s, https_enabled: %s, JSON: %s' % ( ord_url, str(https_enabled), data_to_send,)) + logger.info('Notifying ranger-agent ...') + # set up json and files to be sent to ranger-agent + files = { + 'json': (None, json.dumps(data_to_send), 'application/json'), + 'file': ('heat_template', base64.b64encode(resource_template_data.encode()), + 'application/yaml')} - logger.info('Notifying ORD...') if https_enabled: if conf.ordupdate.cert_path == '': extra_message = '(not using certificate)' @@ -122,16 +149,14 @@ def _notify(ord_url, logger.debug('switch to https, notifying ord_url: %s' % ( ord_url)) try: - # Added the header to support the older version of requests - headers = {'Content-Type': 'application/json'} if not conf.ordupdate.cert_path: response = requests.post('%s/v1/ord/ord_notifier' % (ord_url,), - data=json.dumps(data_to_send), + files=files, headers=headers, verify=conf.verify) else: response = requests.post('%s/v1/ord/ord_notifier' % (ord_url,), - data=json.dumps(data_to_send), + files=files, headers=headers, cert=conf.ordupdate.cert_path) except requests.exceptions.SSLError: @@ -142,10 +167,10 @@ def _notify(ord_url, ord_url = 'http%s' % ord_url[5:] logger.debug('https not supported, notifying ord_url: %s' % ( ord_url)) - headers = {'Content-Type': 'application/json'} + response = requests.post('%s/v1/ord/ord_notifier' % (ord_url,), - headers=headers, - data=json.dumps(data_to_send)) + files=files, + headers=headers) # Make sure the ORD sent an ACK if response.status_code != ACK_CODE: @@ -169,6 +194,18 @@ def _update_audit(lcp_name, application_id, tracking_id, transaction_id, logger.info('LCP %s: %s (%s)' % (lcp_name, event_details, status, )) +# new for template data table +def _update_template_data(transaction_id, resource_name, + resource_template_data): + + data_to_save = dict( + transaction_id=transaction_id, + stack_name=resource_name, + stack_template=resource_template_data) + + regionResourceIdStatus.add_update_template_data(data_to_save) + + def _update_resource_status(region, resource_id, status, transaction_id, error_code, error_msg, resource_operation, resource_type): @@ -190,7 +227,7 @@ def _update_resource_status(region, resource_id, status, transaction_id, resource_type=resource_type, ord_notifier_id="") - regionResourceIdStatus.add_status(data_to_save) + return regionResourceIdStatus.add_status(data_to_save) def notify_ord(transaction_id, @@ -201,12 +238,12 @@ def notify_ord(transaction_id, resource_id, operation, region_id, + resource_template_data, application_id, user_id, external_id=None, - error=False, headers={}): - """Notify ORD of the changes. + """Notify ranger agent of the changes. This function should be called after a resource has changed in SoT (created, modified or deleted). @@ -224,65 +261,73 @@ def notify_ord(transaction_id, :param application_id: The running application ID (RDS, CMS, etc.) :param user_id: The calling user ID :param external_id: An external tracking ID (optional) - :param error: A boolean that says whether an error has occurred during the - upload operation :return: :raise: ConfigFileError - when the configuration file was not found, - OrdNotFoundError - when the ORD was not found, + OrdNotFoundError - when the ranger agent URL not found, requests.exceptions.ConnectionError when the POST request cannot be sent, - NotifyNotAcknowledgedError - when the ORD did not respond to the + NotifyNotAcknowledgedError - when ranger agent did not respond to the notification as expected """ logger.debug('Entered notify_ord with transaction_id: %s, ' 'tracking_id: %s, resource_type: %s, ' 'resource_template_version: %s, resource_name: %s, ' 'resource_id: %s, operation: %s, region_id: %s, ' - 'application_id: %s, user_id: %s, external_id: %s, ' - 'error: %s' % (transaction_id, tracking_id, resource_type, - resource_template_version, resource_name, - resource_id, operation, region_id, - application_id, user_id, external_id, error,)) + 'application_id: %s, user_id: %s, external_id: %s, ' % ( + transaction_id, tracking_id, resource_type, + resource_template_version, resource_name, + resource_id, operation, region_id, + application_id, user_id, external_id,)) error_msg = '' transaction_type = '%s %s' % (operation, resource_type, ) try: - if error: - event_details = 'upload failed' - status = 'SoT_Error' - error_msg = 'Upload to SoT Git repository failed' - else: - # Discover the correct ORD - discover_url = '%s:%d' % (conf.ordupdate.discovery_url, - conf.ordupdate.discovery_port,) - ord_to_update = _find_correct_ord(discover_url, region_id, headers) + # Discover the ranger agent url + discover_url = '%s:%d' % (conf.ordupdate.discovery_url, + conf.ordupdate.discovery_port,) + ord_to_update = _find_correct_ord(discover_url, region_id, headers) - if ord_to_update is None: - message = 'ORD of LCP %s not found' % (region_id, ) - logger.error(message) - raise OrdNotFoundError(message) + if ord_to_update is None: + message = 'Ranger Agent URL of LCP %s not found' % (region_id, ) + logger.error(message) + raise OrdNotFoundError(Exception(message)) - _notify(ord_to_update, - transaction_id, - resource_id, - resource_type, - resource_template_version, - resource_name, - operation, - region_id) + if operation != 'create': + record = regionResourceIdStatus.get_template_by_trans_id( + resource_id, region_id) + if operation == 'modify': + resource_template_version = record.template_version + 1 - # All OK - event_details = '%s notified' % (region_id, ) - status = 'Success' + _notify(ord_to_update, + transaction_id, + resource_id, + resource_type, + str(resource_template_version), + resource_name, + operation, + region_id, + resource_template_data + ) + + # All OK + event_details = '%s notified' % (region_id, ) + status = 'Success' + except NoTokenError: + event_details = 'GET token failed for region %s' % (region_id,) + status = 'GET Token Error' + error_msg = 'Failed to get token for region %s' % (region_id,) + raise except Exception: event_details = '%s notification failed' % (region_id, ) status = 'ORD_Error' - error_msg = 'Notification to ORD failed' + error_msg = 'Notification to Ranger Agent failed' raise finally: # Update resource_status db with status - _update_resource_status(region_id, resource_id, status, transaction_id, - 0, error_msg, operation, resource_type) + resource_transaction_id = \ + _update_resource_status(region_id, resource_id, status, + transaction_id, 0, error_msg, + operation, resource_type) # Write a record to Audit repository. Note that I assigned the # appropriate values to event_details and status in every flow, so @@ -290,6 +335,11 @@ def notify_ord(transaction_id, _update_audit(region_id, application_id, tracking_id, transaction_id, transaction_type, resource_id, user_id, external_id, event_details, status) - logger.debug( - "Create Resource Requested to ORD: region=%s resource_id=%s status=%s" % ( - region_id, resource_id, status)) + if operation in ('create', 'modify'): + # add entry to resource_template_data table + _update_template_data(resource_transaction_id, resource_name, + resource_template_data) + + logger.debug( + "Create Resource Requested to ranger agent: region=%s resource_id=%s " + "status=%s" % (region_id, resource_id, status)) diff --git a/orm/services/resource_distributor/rds/services/model/region_resource_id_status.py b/orm/services/resource_distributor/rds/services/model/region_resource_id_status.py index e3f18c69..3a229a69 100755 --- a/orm/services/resource_distributor/rds/services/model/region_resource_id_status.py +++ b/orm/services/resource_distributor/rds/services/model/region_resource_id_status.py @@ -10,7 +10,7 @@ class ResourceMetaData(object): return self.__dict__ -class Model(object): +class ResourceStatusModel(object): def __init__(self, timestamp, region, @@ -45,6 +45,21 @@ class Model(object): return self.__dict__ +class ResourceTemplateModel(object): + def __init__(self, + transaction_id, + resource_name, + template_version, + template_data): + self.transaction_id = transaction_id + self.resource_name = resource_name + self.template_version = template_version + self.template_data = template_data + + def as_dict(self): + return self.__dict__ + + class StatusModel(object): def __init__(self, status): self.regions = status diff --git a/orm/services/resource_distributor/rds/services/region_resource_id_status.py b/orm/services/resource_distributor/rds/services/region_resource_id_status.py index 53ee5322..18c66a6e 100755 --- a/orm/services/resource_distributor/rds/services/region_resource_id_status.py +++ b/orm/services/resource_distributor/rds/services/region_resource_id_status.py @@ -16,6 +16,42 @@ num_of_seconds_in_minute = 60 num_of_miliseconds_in_seconds = 1000 +def add_update_template_data(data): + logger.debug("update template stack name [{}], and " + "transaction_id [{}] ".format(data['stack_name'], + data['transaction_id'])) + try: + conn = factory.get_resource_stack_data_connection() + conn.add_update_template_record(data['transaction_id'], + data['stack_name'], + data['stack_template']) + + except Exception: + logger.exception("Unexpected error: {}".format(sys.exc_info()[0])) + raise + + +def delete_resource_status_data(resource_id): + # delete all resource status entries for the given resource_id + pass + + +def get_template_by_trans_id(resource_id, region): + logger.debug("get template transaction id for resource %s and " + "region %s" % (resource_id, region)) + + try: + conn = factory.get_region_resource_id_status_connection() + resource_trans_id = conn.get_resource_region_data(resource_id, region) + conn = factory.get_resource_stack_data_connection() + record = conn.get_resource_template_data(resource_trans_id) + return record + + except Exception: + logger.exception("Unexpected error: {}".format(sys.exc_info()[0])) + raise + + def add_status(data): logger.debug("add resource status timestamp [{}], region [{}], status [{}] " ", transaction_id [{}] and resource_id [{}], ord_notifier_id [{}], " @@ -36,12 +72,15 @@ def add_status(data): validate_resource_type(data['resource_type']) conn = factory.get_region_resource_id_status_connection() - conn.add_update_status_record(data['timestamp'], data['region'], data['status'], - data['transaction_id'], data['resource_id'], - data['ord_notifier_id'], data['error_msg'], - data['error_code'], - data['resource_operation'], - data.get('resource_extra_metadata')) + + resource_transaction_id = \ + conn.add_update_status_record( + data['timestamp'], data['region'], data['status'], + data['transaction_id'], data['resource_id'], + data['ord_notifier_id'], data['error_msg'], + data['error_code'], data['resource_operation'], + data.get('resource_extra_metadata')) + return resource_transaction_id # post_data_to_image(data) except Error as e: logger.exception("invalid inputs error") @@ -66,6 +105,15 @@ def get_regions_by_status_resource_id(status, resource_id): return result +def delete_resource_template_data(resource_id, region): + logger.debug("delete resource_template_data for resource %s and " + "region %s" % (resource_id, region)) + conn = factory.get_region_resource_id_status_connection() + resource_trans_id = conn.get_resource_region_data(resource_id, region) + conn = factory.get_resource_stack_data_connection() + conn.delete_resource_template(resource_trans_id) + + def validate_resource_type(resource_type): allowed_resource_type = config['allowed_resource_type'] if resource_type not in allowed_resource_type: diff --git a/orm/services/resource_distributor/rds/services/resource.py b/orm/services/resource_distributor/rds/services/resource.py index 83075596..f5139966 100755 --- a/orm/services/resource_distributor/rds/services/resource.py +++ b/orm/services/resource_distributor/rds/services/resource.py @@ -1,15 +1,17 @@ """create resource moudle.""" import logging +import threading import time from orm.services.flavor_manager.fms_rest.data.sql_alchemy.data_manager \ import DataManager +from orm.services.resource_distributor.rds.ordupdate.ord_notifier \ + import notify_ord, NoTokenError from orm.services.resource_distributor.rds.services import region_resource_id_status as regionResourceIdStatus from orm.services.resource_distributor.rds.services import (yaml_customer_builder, yaml_flavor_builder, yaml_group_builder, yaml_image_builder) from orm.services.resource_distributor.rds.services.base import ConflictValue, ErrorMessage from orm.services.resource_distributor.rds.services.model.resource_input import ResourceData as InputData -from orm.services.resource_distributor.rds.sot import sot_factory from orm.services.resource_distributor.rds.utils import utils, uuid_utils @@ -53,7 +55,8 @@ def _create_or_update_resource_status(input_data, target, error_msg='', # check rms region status if not _region_valid(target): status = 'Error' - error_msg = "Not sent to ord as status equal to " + target['rms_status'] + error_msg = "Not sent to ord as status equal to " + \ + target['rms_status'] raise ErrorMessage("Not sent to ord as status equal to %s" % target['rms_status']) @@ -102,7 +105,8 @@ def get_valid_tenants(tenants, region): return valid_tenants_list -def _create_data_to_sot(input_data): +def _create_template_data(input_data): + """create data. : build yaml string @@ -158,14 +162,59 @@ def _create_data_to_sot(input_data): return targetslist -def _upload_to_sot(uuid, tranid, targetslist): +def send_data_to_ord(tracking_id, transaction_id, resource_list, + application_id, user_id, headers={}): + my_logger.info("Prepping data to send to ord ...") + + # This method is called also in case exception raised. + # Notification to ords will not be sent but status db and audit + # will be updated. + for resource in resource_list: + try: + resource_stack_name = 's_{}'.format(resource["resource_name"]) + notify_ord(transaction_id, + tracking_id, + resource["resource_type"], + 0, # This is the resource-template-version + resource_stack_name, # This is the stack name + resource["resource_name"], # This is the resource_id + resource["operation"], + resource["region_id"], + resource["template_data"], + application_id, # application_id is not available + user_id, # user_id is not available + "NA", # external_id is not available + headers) + except NoTokenError as e: + my_logger.error("Failed to get token for resource id {} " + "region {}".format(resource["resource_name"], + resource["region_id"])) + except Exception as e: + my_logger.error("Error in updating ORD! Error: {}".format( + str(e) + )) + + +def _save_resource_to_ord(tracking_id, transaction_id, + resource_list, application_id, user_id, + headers={}): + thread = threading.Thread(target=send_data_to_ord, + args=(tracking_id, + transaction_id, + resource_list, + application_id, + user_id, + headers)) + thread.start() + + +def _submit_template_data(uuid, tranid, targetslist): application_id = request.headers[ 'X-RANGER-Client'] if 'X-RANGER-Client' in request.headers else \ 'NA' user_id = request.headers[ 'X-RANGER-Requester'] if 'X-RANGER-Requester' in request.headers else \ '' - sot = sot_factory.get_sot() headers = {} headers['X-Auth-Region'] = request.headers[ 'X-Auth-Region'] if 'X-Auth-Region' in \ @@ -173,12 +222,13 @@ def _upload_to_sot(uuid, tranid, targetslist): headers['X-Auth-Token'] = request.headers[ 'X-Auth-Token'] if 'X-Auth-Token' in \ request.headers else '' - sot.save_resource_to_sot(uuid, - tranid, - targetslist, - application_id, - user_id, - headers) + + _save_resource_to_ord(uuid, + tranid, + targetslist, + application_id, + user_id, + headers) def _check_resource_status(input_data): @@ -193,15 +243,14 @@ def _check_resource_status(input_data): raise ConflictValue([region.region for region in regions_by_resource.regions]) -def update_sot(input_data): +def _generate_resource_data(input_data): """create resource.""" my_logger.debug("build yaml file for %s id: %s" % (input_data.resource_type, input_data.resource_id)) - targetslist = _create_data_to_sot(input_data) - my_logger.debug("upload yaml to SoT") - _upload_to_sot(input_data.resource_id, - input_data.transaction_id, - targetslist) + targetslist = _create_template_data(input_data) + my_logger.debug("submit yaml to ranger-agent...") + _submit_template_data(input_data.resource_id, input_data.transaction_id, + targetslist) def main(jsondata, external_transaction_id, resource_type, operation): @@ -222,7 +271,7 @@ def main(jsondata, external_transaction_id, resource_type, operation): # add regions status from rms (to check if it down) input_data.targets = utils.add_rms_status_to_regions( input_data.targets, input_data.resource_type) - update_sot(input_data) + _generate_resource_data(input_data) except ConflictValue: raise except ErrorMessage as exp: diff --git a/orm/services/resource_distributor/rds/sot/__init__.py b/orm/services/resource_distributor/rds/sot/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/orm/services/resource_distributor/rds/sot/git_sot/__init__.py b/orm/services/resource_distributor/rds/sot/git_sot/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/orm/services/resource_distributor/rds/sot/sot_factory.py b/orm/services/resource_distributor/rds/sot/sot_factory.py deleted file mode 100644 index e0d9fca2..00000000 --- a/orm/services/resource_distributor/rds/sot/sot_factory.py +++ /dev/null @@ -1,29 +0,0 @@ -from orm.services.resource_distributor.rds.sot.git_sot import git_sot - -sot_type = "" -local_repository_path = "" -relative_path_format = "" -file_name_format = "" -commit_message_format = "" -commit_user = "" -commit_email = "" -git_server_url = "" -git_type = "" - - -def get_sot(): - """Return the correct SoT implementation according to sot_type""" - - if sot_type == 'git': - git_sot.GitSoT.local_repository_path = local_repository_path - git_sot.GitSoT.relative_path_format = relative_path_format - git_sot.GitSoT.file_name_format = file_name_format - git_sot.GitSoT.commit_message_format = commit_message_format - git_sot.GitSoT.commit_user = commit_user - git_sot.GitSoT.commit_email = commit_email - git_sot.GitSoT.git_server_url = git_server_url - git_sot.GitSoT.git_type = git_type - sot = git_sot.GitSoT() - return sot - else: - raise RuntimeError("Invalid SoT implementation!!") diff --git a/orm/services/resource_distributor/rds/storage/factory.py b/orm/services/resource_distributor/rds/storage/factory.py index b7e4e938..5924a3cc 100644 --- a/orm/services/resource_distributor/rds/storage/factory.py +++ b/orm/services/resource_distributor/rds/storage/factory.py @@ -1,5 +1,9 @@ -from orm.services.resource_distributor.rds.storage.mysql.region_resource_id_status import \ - Connection as RegionResourceIdStatusConnection +from orm.services.resource_distributor.rds.storage.mysql.region_resource_id_status \ + import ResStatusConnection + +from orm.services.resource_distributor.rds.storage.mysql.region_resource_id_status \ + import ResTemplateConnection + database = { 'url': 'na' @@ -7,4 +11,8 @@ database = { def get_region_resource_id_status_connection(): - return RegionResourceIdStatusConnection(database['url']) + return ResStatusConnection(database['url']) + + +def get_resource_stack_data_connection(): + return ResTemplateConnection(database['url']) diff --git a/orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py b/orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py index 8eb344f6..fd2a69b3 100755 --- a/orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py +++ b/orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py @@ -6,9 +6,9 @@ from oslo_db.sqlalchemy.enginefacade import LegacyEngineFacade from pecan import conf from orm.services.resource_distributor.rds.services.model.region_resource_id_status \ - import Model, StatusModel, RegionEndPointData + import ResourceStatusModel, StatusModel, RegionEndPointData from orm.services.resource_distributor.rds.storage import region_resource_id_status -from sqlalchemy import BigInteger, Column, ForeignKey, Integer, String, Text +from sqlalchemy import BigInteger, BLOB, Column, ForeignKey, Integer, String, Text from sqlalchemy.ext.declarative.api import declarative_base from sqlalchemy.orm import relationship from sqlalchemy.sql import and_ @@ -64,7 +64,19 @@ class RegionEndPoint(Base): return RegionEndPointData(region, url, atype) -class Connection(region_resource_id_status.Base): +class ResourceTemplateRecord(Base): + __tablename__ = 'resource_template_data' + + id = Column(Integer, autoincrement=True, primary_key=True) + transaction_id = Column(Text, + ForeignKey('resource_status.transaction_id'), + primary_key=False) + resource_name = Column(Text, primary_key=False) + template_version = Column(Integer, primary_key=False) + template_data = Column(BLOB, primary_key=False) + + +class ResStatusConnection(region_resource_id_status.ResourceStatusBase): """ Implements mysql DB """ def __init__(self, url): @@ -112,7 +124,6 @@ class Connection(region_resource_id_status.Base): record.timestamp = timestamp record.region = region record.status = status - record.transaction_id = transaction_id record.resource_id = resource_id record.ord_notifier = ord_notifier record.err_msg = err_msg @@ -126,6 +137,7 @@ class Connection(region_resource_id_status.Base): # remove child if not given session.query(ImageMetadData).filter_by( image_meta_data_id=record.id).delete() + return record.transaction_id else: logger.debug("Add record") resource_status = ResourceStatusRecord(timestamp=timestamp, @@ -141,6 +153,7 @@ class Connection(region_resource_id_status.Base): resource_status.resource_extra_metadata.append(image_metadata) session.add(resource_status) + return transaction_id except oslo_db.exception.DBDuplicateEntry as e: logger.warning("Duplicate entry: {}".format(str(e))) @@ -148,6 +161,22 @@ class Connection(region_resource_id_status.Base): def get_records_by_resource_id(self, resource_id): return self.get_records_by_filter_args(resource_id=resource_id) + def get_resource_region_data(self, resource_id, region): + logger.debug("Get resource data by resource_id '{}' and " + "region '{}'".format(resource_id, region)) + try: + session = self._engine_facade.get_session() + with session.begin(): + record = session.query(ResourceStatusRecord).\ + filter_by(resource_id=resource_id, region=region).first() + if record is None: + logger.exception( + 'No resource status record with resource id {} and' + 'region {} found'.format(resource_id, region)) + return record.transaction_id + except Exception as exp: + raise + def get_records_by_filter_args(self, **filter_args): logger.debug("Get records filtered by [{}]".format(filter_args)) (timestamp, ref_timestamp) = self.get_timestamp_pair() @@ -164,16 +193,18 @@ class Connection(region_resource_id_status.Base): record.status = "Error" record.err_msg = "Status updated to 'Error'. Too long 'Submitted' status" - status = Model(record.timestamp, - record.region, - record.status, - record.transaction_id, - record.resource_id, - record.ord_notifier, - record.err_msg, - record.err_code, - record.operation, - record.resource_extra_metadata) + status = ResourceStatusModel( + record.timestamp, + record.region, + record.status, + record.transaction_id, + record.resource_id, + record.ord_notifier, + record.err_msg, + record.err_code, + record.operation, + record.resource_extra_metadata) + records_model.append(status) return StatusModel(records_model) else: @@ -206,16 +237,16 @@ class Connection(region_resource_id_status.Base): record.status = "Error" record.err_msg = "Status updated to 'Error'. Too long 'Submitted' status" else: - status = Model(record.timestamp, - record.region, - record.status, - record.transaction_id, - record.resource_id, - record.ord_notifier, - record.err_msg, - record.err_code, - record.operation, - record.resource_extra_metadata) + status = ResourceStatusModel(record.timestamp, + record.region, + record.status, + record.transaction_id, + record.resource_id, + record.ord_notifier, + record.err_msg, + record.err_code, + record.operation, + record.resource_extra_metadata) records_model.append(status) if len(records_model): return StatusModel(records_model) @@ -244,10 +275,79 @@ class Connection(region_resource_id_status.Base): RegionEndPoint.end_point_type == key_ep)) if record.first(): - # return record.first().to_wsme() return record.first().public_url return None except Exception as exp: logger.exception("DB error RegionEndPoint filtering by region name") raise + + +class ResTemplateConnection(region_resource_id_status.ResourceTemplateBase): + + def __init__(self, url): + self._engine_facade = LegacyEngineFacade(url) + + def add_update_template_record(self, + transaction_id, + resource_name, + resource_template): + + logger.debug("Add/Update template record:\ntransaction_id [{}]\n" + "resource name [{}]\n\n".format(transaction_id, + resource_name)) + try: + session = self._engine_facade.get_session() + with session.begin(): + + record = session.query(ResourceTemplateRecord).\ + filter_by(transaction_id=transaction_id).first() + if record is not None: + logger.debug("Update resource template record") + record.resource_name = resource_name + record.template_version = record.template_version + 1 + record.template_data = resource_template + else: + logger.debug("Add resource template record") + resource_template_record = \ + ResourceTemplateRecord( + transaction_id=transaction_id, + resource_name=resource_name, + template_version=0, + template_data=resource_template.encode("utf-8")) + + session.add(resource_template_record) + + except oslo_db.exception.DBDuplicateEntry as e: + logger.warning("Duplicate entry: {}".format(str(e))) + + def delete_resource_template(self, trans_id): + try: + session = self._engine_facade.get_session() + with session.begin(): + record = session.query(ResourceTemplateRecord).\ + filter_by(transaction_id=trans_id).delete() + if record is None: + logger.exception( + 'Template data not found with transaction' + 'id {}'.format(trans_id)) + + except Exception as exp: + raise + + def get_resource_template_data(self, transaction_id): + logger.debug("Get resource template data by transaction " + "id '{}' ".format(transaction_id)) + + try: + session = self._engine_facade.get_session() + with session.begin(): + record = session.query(ResourceTemplateRecord).\ + filter_by(transaction_id=transaction_id).first() + if record is None: + logger.exception( + 'No resource template record found with transaction ' + 'id '.format(transaction_id)) + return record + except Exception as exp: + raise diff --git a/orm/services/resource_distributor/rds/storage/region_resource_id_status.py b/orm/services/resource_distributor/rds/storage/region_resource_id_status.py index b2a65c53..e72a5876 100644 --- a/orm/services/resource_distributor/rds/storage/region_resource_id_status.py +++ b/orm/services/resource_distributor/rds/storage/region_resource_id_status.py @@ -2,7 +2,7 @@ """ -class Base(object): +class ResourceStatusBase(object): def __init__(self, url): pass @@ -15,10 +15,27 @@ class Base(object): ord_notifier, err_msg, err_code): - raise NotImplementedError("Please Implement this method") + raise NotImplementedError(Exception("Please Implement this method")) def get_records_by_resource_id(self, resource_id): - raise NotImplementedError("Please Implement this method") + raise NotImplementedError(Exception("Please Implement this method")) def get_records_by_filter_args(self, **filter_args): - raise NotImplementedError("Please Implement this method") + raise NotImplementedError(Exception("Please Implement this method")) + + +class ResourceTemplateBase(object): + def __init__(self, url): + pass + + def add_update_template_record(self, + transaction_id, + stack_name, + stacK_template): + raise NotImplementedError(Exception("Please Implement this method")) + + def get_records_by_transaction_id(self, transaction_id): + raise NotImplementedError(Exception("Please Implement this method")) + + def get_trans_by_filter_args(self, **filter_args): + raise NotImplementedError(Exception("Please Implement this method")) diff --git a/orm/services/resource_distributor/rds/utils/utils.py b/orm/services/resource_distributor/rds/utils/utils.py index d80c0888..69659e6b 100755 --- a/orm/services/resource_distributor/rds/utils/utils.py +++ b/orm/services/resource_distributor/rds/utils/utils.py @@ -3,8 +3,12 @@ import logging import re import requests -from orm.services.resource_distributor.rds.proxies import rds_resource_service_proxy -from orm.services.resource_distributor.rds.services.base import ErrorMessage +from orm.services.resource_distributor.rds.proxies import \ + rds_resource_service_proxy +from orm.services.resource_distributor.rds.services.base \ + import ErrorMessage +from orm.services.resource_distributor.rds.services import \ + region_resource_id_status as regionResourceIdStatus from pecan import conf, request @@ -28,6 +32,9 @@ def invoke_delete_region(data): rds_resource_service_proxy.invoke_resources_region_delete( resource_type=data['resource_type'], resource_id=data['resource_id'], region=data['region']) + # delete heat template entry + regionResourceIdStatus.delete_resource_template_data( + data['resource_id'], data['region']) return @@ -74,7 +81,7 @@ def add_rms_status_to_regions(resource_regions, resource_type): supported_resource_version = [value for key, value in supported_versions if key == resource_type] - # iterate through rms regions and gett regions status and version + # iterate through rms regions and get regions status and version for region in all_regions['regions']: rms_regions[region['name']] = {'status': region['status'], 'version': region['rangerAgentVersion']} diff --git a/orm/services/resource_distributor/scripts/db_scripts/create_db.sql b/orm/services/resource_distributor/scripts/db_scripts/create_db.sql index 9746722f..dac4dd73 100755 --- a/orm/services/resource_distributor/scripts/db_scripts/create_db.sql +++ b/orm/services/resource_distributor/scripts/db_scripts/create_db.sql @@ -10,17 +10,35 @@ use orm; create table if not exists resource_status ( id integer auto_increment not null, - timestamp bigint not null, - region varchar(64) not null, - resource_id varchar(64) not null, - status varchar(16) not null, - transaction_id varchar(64), - ord_notifier varchar(64) not null, - err_msg varchar(255), - err_code varchar(64), - operation varchar(64), - primary key (id), - unique(resource_id, region)); + timestamp bigint not null, + region varchar(64) not null, + resource_id varchar(64) not null, + status varchar(16) not null, + transaction_id varchar(64), + ord_notifier varchar(64) not null, + err_msg varchar(255), + err_code varchar(64), + operation varchar(64), + primary key (id), + unique(resource_id, region), + unique transaction_idx (transaction_id)); + + +#***** +#* MySql script for Creating Table resource_template_data +#***** + +create table if not exists resource_template_data +( + id integer auto_increment not null, + transaction_id varchar(64), + resource_name varchar(64) NOT NULL, + template_version integer, + template_data BLOB NOT NULL, + primary key (id), + foreign key (transaction_id) references resource_status(transaction_id) ON DELETE CASCADE +); + #***** #* MySql script for Creating Table image_metadata diff --git a/orm/tests/unit/rds/controllers/v1/status/test_get_resource_status.py b/orm/tests/unit/rds/controllers/v1/status/test_get_resource_status.py index 7f07661a..f4fa02bf 100755 --- a/orm/tests/unit/rds/controllers/v1/status/test_get_resource_status.py +++ b/orm/tests/unit/rds/controllers/v1/status/test_get_resource_status.py @@ -1,7 +1,7 @@ """unittest get resource status.""" from mock import MagicMock import orm.services.resource_distributor.rds.controllers.v1.status.get_resource as resource -from orm.services.resource_distributor.rds.services.model.region_resource_id_status import Model, StatusModel +from orm.services.resource_distributor.rds.services.model.region_resource_id_status import ResourceStatusModel, StatusModel from orm.tests.unit.rds.controllers.v1.functional_test import FunctionalTest @@ -31,7 +31,7 @@ class GetResourceStatus(FunctionalTest): def test_get_valid_resource(self): """get valid resource.""" - result = Model( + result = ResourceStatusModel( status="200", timestamp="123456789", region="name", transaction_id=5, resource_id="1", ord_notifier="", err_msg="123", err_code="12", operation="create" diff --git a/orm/tests/unit/rds/ordupdate/test_ord_notifier.py b/orm/tests/unit/rds/ordupdate/test_ord_notifier.py index 83d8bdba..9cf3950c 100755 --- a/orm/tests/unit/rds/ordupdate/test_ord_notifier.py +++ b/orm/tests/unit/rds/ordupdate/test_ord_notifier.py @@ -96,51 +96,97 @@ class MainTest(unittest.TestCase): self.assertEqual('test', result) @mock.patch.object(ord_notifier, 'conf') - @mock.patch.object(ord_notifier.json, 'dumps') - def test_notify_sanity(self, mock_dumps, mock_conf): - ord_notifier.requests.post = mock.MagicMock( - return_value=MyResponse(ord_notifier.ACK_CODE, None)) - ord_notifier._notify(*("1",) * 8) + @mock.patch.object(ord_notifier, 'AuthService') + def test_token_not_found(self, mock_authentication, mock_conf): + token_id = None + region = 'local' + mock_conf.authentication.enabled = True + mock_authentication.get_token = mock.MagicMock( + return_value=token_id) + self.assertRaises(ord_notifier.NoTokenError) @mock.patch.object(ord_notifier, 'conf') + @mock.patch.object(ord_notifier, 'AuthService') @mock.patch.object(ord_notifier.json, 'dumps') - def test_notify_not_acknowledged(self, mock_dumps, mock_conf): + def test_notify_sanity(self, mock_dumps, mock_authentication, mock_conf): + token_id = 'ffff' + region = 'local' + mock_conf.authentication.enabled = True + mock_authentication.get_token = mock.MagicMock( + return_value=token_id) + + ord_notifier.requests.post = mock.MagicMock( + return_value=MyResponse(ord_notifier.ACK_CODE, None)) + ord_notifier._notify(*("1",) * 9) + + @mock.patch.object(ord_notifier, 'conf') + @mock.patch.object(ord_notifier, 'AuthService') + @mock.patch.object(ord_notifier.json, 'dumps') + def test_notify_not_acknowledged(self, mock_dumps, mock_authentication, + mock_conf): + token_id = 'ffff' + region = 'local' + mock_conf.authentication.enabled = True + mock_authentication.get_token = mock.MagicMock( + return_value=token_id) ord_notifier.requests.post = mock.MagicMock( return_value=MyResponse(404, None)) try: - ord_notifier._notify(*("1",) * 8) + ord_notifier._notify(*("1",) * 9) self.fail('notify() passed successfully' '(expected NotifyNotAcknowledgedError)') except ord_notifier.NotifyNotAcknowledgedError: pass @mock.patch.object(ord_notifier, 'conf') - def test_notify_https_disabled_but_received(self, mock_conf): + @mock.patch.object(ord_notifier, 'AuthService') + def test_notify_https_disabled_but_received(self, mock_authentication, + mock_conf): + token_id = 'ffff' + region = 'local' + mock_conf.authentication.enabled = True + mock_authentication.get_token = mock.MagicMock( + return_value=token_id) ord_notifier.requests.post = validate_http_post mock_conf.ordupdate.https_enabled = False mock_conf.ordupdate.template_type = 'a' - ord_notifier._notify('https://127.0.0.1:1337', * ("1", ) * 7) + ord_notifier._notify('https://127.0.0.1:1337', * ("1", ) * 8) @mock.patch.object(ord_notifier, 'conf') + @mock.patch.object(ord_notifier, 'AuthService') @mock.patch.object(ord_notifier.json, 'dumps') def test_notify_https_enabled_and_no_certificate(self, mock_dumps, + mock_authentication, mock_conf): + token_id = 'ffff' + region = 'local' + mock_conf.authentication.enabled = True + mock_authentication.get_token = mock.MagicMock( + return_value=token_id) ord_notifier.requests.post = validate_https_post mock_conf.ordupdate.https_enabled = True mock_conf.ordupdate.cert_path = '' - ord_notifier._notify('https://127.0.0.1:1337', *("1",) * 7) + ord_notifier._notify('https://127.0.0.1:1337', *("1",) * 8) @mock.patch.object(ord_notifier, 'conf') + @mock.patch.object(ord_notifier, 'AuthService') @mock.patch.object(ord_notifier.json, 'dumps') - def test_notify_https_enabled_and_ssl_error(self, mock_dumps, mock_conf): + def test_notify_https_enabled_and_ssl_error(self, mock_dumps, + mock_authentication, + mock_conf): + token_id = 'ffff' + region = 'local' + mock_conf.authentication.enabled = True + mock_authentication.get_token = mock.MagicMock( + return_value=token_id) ord_notifier.requests.post = mock.MagicMock( side_effect=ord_notifier.requests.exceptions.SSLError('test')) mock_conf.ordupdate.https_enabled = True mock_conf.ordupdate.cert_path = '' self.assertRaises(ord_notifier.requests.exceptions.SSLError, ord_notifier._notify, 'https://127.0.0.1:1337', - *("1",) * 7) + *("1",) * 8) @patch.object(ord_notifier.audit, 'audit') @patch.object(ord_notifier, 'regionResourceIdStatus') @@ -150,23 +196,22 @@ class MainTest(unittest.TestCase): return_value=MyResponse(404, 'test')) try: ord_notifier.notify_ord('test', '1', '2', '3', '4', '5', '6', - 'gigi', '7', '') + 'gigi', '7', '', '') self.fail('notify_ord() passed successfully (expected OrdNotFoundError)') except ord_notifier.OrdNotFoundError as e: - self.assertEqual(str(e), 'ORD of LCP %s not found' % ( + self.assertEqual(str(e), 'Ranger Agent URL of LCP %s not found' % ( 'gigi', )) @patch.object(ord_notifier.audit, 'audit') @patch.object(ord_notifier, 'regionResourceIdStatus') @mock.patch.object(ord_notifier, 'conf') - def test_main_error(self, mock_audit, mock_region, mock_conf): + def test_main_ord_not_found(self, mock_audit, mock_region, mock_conf): ord_notifier.requests.get = mock.MagicMock( - return_value=MyResponse(ord_notifier.OK_CODE, - {'regions': [{'endpoints': [ - {'publicurl': 'test', - 'type': 'ord'}]}]})) - ord_notifier.requests.post = mock.MagicMock( - return_value=MyResponse(ord_notifier.ACK_CODE, None)) - - ord_notifier.notify_ord('test', '1', '2', '3', '4', '5', '6', '7', - '8', '9', '10', True) + return_value=MyResponse(404, 'test')) + try: + ord_notifier.notify_ord('test', '1', '2', '3', '4', '5', '6', + 'gigi', '7', '', '') + self.fail('notify_ord() passed successfully (expected OrdNotFoundError)') + except ord_notifier.OrdNotFoundError as e: + self.assertEqual(str(e), 'Ranger Agent URL of LCP %s not found' % ( + 'gigi', )) diff --git a/orm/tests/unit/rds/services/model/test_region_resource_id_status.py b/orm/tests/unit/rds/services/model/test_region_resource_id_status.py index b9697d50..0c088140 100755 --- a/orm/tests/unit/rds/services/model/test_region_resource_id_status.py +++ b/orm/tests/unit/rds/services/model/test_region_resource_id_status.py @@ -5,8 +5,8 @@ from orm.services.resource_distributor.rds.services.model import region_resource class TestModel(unittest.TestCase): def test_model_as_dict(self): - model = region_resource_id_status.Model(1, 2, 3, 4, 5, 6, 7, 8, - 'create') + model = region_resource_id_status.ResourceStatusModel(1, 2, 3, 4, 5, 6, 7, 8, + 'create') expected_dict = { 'timestamp': 1, 'region': 2, @@ -24,21 +24,36 @@ class TestModel(unittest.TestCase): self.assertEqual(test_dict, expected_dict) +class TestResourceTemplateModel(unittest.TestCase): + def test_model_as_dict(self): + model = region_resource_id_status.ResourceTemplateModel(1, 2, 3, 4) + + expected_template_dict = { + 'transaction_id': 1, + 'resource_name': 2, + 'template_version': 3, + 'template_data': 4 + } + + test_dict = model.as_dict() + self.assertEqual(test_dict, expected_template_dict) + + class TestStatusModel(unittest.TestCase): def test_get_aggregated_status_error(self): - model = region_resource_id_status.Model(1, 2, 'Error', 4, 5, 6, 7, 8, - 'create') + model = region_resource_id_status.ResourceStatusModel(1, 2, 'Error', 4, 5, 6, 7, 8, + 'create') status_model = region_resource_id_status.StatusModel([model]) self.assertEqual(status_model.status, 'Error') def test_get_aggregated_status_pending(self): - model = region_resource_id_status.Model(1, 2, 'Submitted', 4, 5, 6, 7, - 8, 'create') + model = region_resource_id_status.ResourceStatusModel(1, 2, 'Submitted', 4, 5, 6, 7, + 8, 'create') status_model = region_resource_id_status.StatusModel([model]) self.assertEqual(status_model.status, 'Pending') def test_get_aggregated_status_success(self): - model = region_resource_id_status.Model(1, 2, 'Success', 4, 5, 6, 7, 8, - 'create') + model = region_resource_id_status.ResourceStatusModel(1, 2, 'Success', 4, 5, 6, 7, 8, + 'create') status_model = region_resource_id_status.StatusModel([model]) self.assertEqual(status_model.status, 'Success') diff --git a/orm/tests/unit/rds/services/test_create_resource.py b/orm/tests/unit/rds/services/test_create_resource.py index 99190a9a..1d6e2338 100755 --- a/orm/tests/unit/rds/services/test_create_resource.py +++ b/orm/tests/unit/rds/services/test_create_resource.py @@ -3,11 +3,11 @@ import unittest from unittest.mock import patch from orm.services.resource_distributor.rds.services import resource as ResourceService -from orm.services.resource_distributor.rds.services.model.region_resource_id_status import (Model, +from orm.services.resource_distributor.rds.services.model.region_resource_id_status import (ResourceStatusModel, ResourceMetaData, StatusModel) -result = Model( +result = ResourceStatusModel( status="success", timestamp="123456789", region="name", transaction_id=5, resource_id="1", ord_notifier="", err_msg="123", err_code="12", operation="create", @@ -43,18 +43,6 @@ class InputData(object): self.external_transaction_id = external_transaction_id -class SoT(object): - """mock class.""" - - def save_resource_to_sot(*args): - """mock function.""" - return None - - def delete_resource_from_sot(*args): - """mock function.""" - return None - - class CreateResource(unittest.TestCase): """create resource test.""" @@ -84,8 +72,8 @@ class CreateResource(unittest.TestCase): @patch.object(ResourceService.regionResourceIdStatus, 'add_status', return_value=None) - @patch.object(ResourceService, '_upload_to_sot', return_value=[1, 2]) - @patch.object(ResourceService, '_create_data_to_sot', return_value=[1, 2]) + @patch.object(ResourceService, '_submit_template_data', return_value=[1, 2]) + @patch.object(ResourceService, '_create_template_data', return_value=[1, 2]) @patch.object(ResourceService.regionResourceIdStatus, 'get_regions_by_status_resource_id', return_value=None) @patch.object(ResourceService.uuid_utils, 'get_random_uuid', @@ -159,12 +147,11 @@ class CreateResource(unittest.TestCase): 'add_status', return_value=None) @patch.object(ResourceService.yaml_customer_builder, 'yamlbuilder', return_value=["anystring"]) - @patch.object(ResourceService.sot_factory, 'get_sot', return_value=SoT()) @patch.object(ResourceService.regionResourceIdStatus, 'get_regions_by_status_resource_id', return_value=None) @patch.object(ResourceService.uuid_utils, 'get_random_uuid', return_value='uuid-gen-123456') - def test_create_flavor_sot_data_check(self, tranid, result, sotupload, + def test_create_flavor_sot_data_check(self, tranid, result, yamlbuilder, database): """check list creating.""" input_data = InputData( @@ -177,7 +164,7 @@ class CreateResource(unittest.TestCase): status_model = StatusModel(status=[result]) status_model.regions = None result.return_value = status_model - result = ResourceService._create_data_to_sot(input_data) + result = ResourceService._create_template_data(input_data) self.assertEqual(result, target_list) # @patch.object(ResourceService.regionResourceIdStatus, @@ -190,8 +177,8 @@ class CreateResource(unittest.TestCase): @patch.object(ResourceService.regionResourceIdStatus, 'add_status', return_value=None) - @patch.object(ResourceService, '_upload_to_sot', return_value=[1, 2]) - @patch.object(ResourceService, '_create_data_to_sot', return_value=[1, 2]) + @patch.object(ResourceService, '_submit_template_data', return_value=[1, 2]) + @patch.object(ResourceService, '_create_template_data', return_value=[1, 2]) @patch.object(ResourceService.regionResourceIdStatus, 'get_regions_by_status_resource_id', return_value=None) @patch.object(ResourceService.uuid_utils, 'get_random_uuid', diff --git a/orm/tests/unit/rds/sot/__init__.py b/orm/tests/unit/rds/sot/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/orm/tests/unit/rds/sot/git_sot/__init__.py b/orm/tests/unit/rds/sot/git_sot/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/orm/tests/unit/rds/sot/test_base_sot.py b/orm/tests/unit/rds/sot/test_base_sot.py deleted file mode 100644 index c21c61ce..00000000 --- a/orm/tests/unit/rds/sot/test_base_sot.py +++ /dev/null @@ -1,18 +0,0 @@ -import unittest - -from orm.services.resource_distributor.rds.sot.base_sot import BaseSoT - - -class BaseSoTTests(unittest.TestCase): - - def test_base_sot_no_method_save_implemented(self): - """ Check if creating an instance and calling save method fail""" - with self.assertRaises(Exception): - sot = BaseSoT() - sot.save_resource_to_sot('1', '2', []) - - def test_base_sot_no_method_validate_implemented(self): - """ Check if creating an instance and calling validate method fail""" - with self.assertRaises(Exception): - sot = BaseSoT() - sot.validate_sot_state() diff --git a/orm/tests/unit/rds/storage/mysql/test_region_resource_id_status.py b/orm/tests/unit/rds/storage/mysql/test_region_resource_id_status.py index f50a5a28..7a07d07d 100755 --- a/orm/tests/unit/rds/storage/mysql/test_region_resource_id_status.py +++ b/orm/tests/unit/rds/storage/mysql/test_region_resource_id_status.py @@ -72,7 +72,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase): """Test that no exception is raised when calling add_update_status_record. where record exist """ - my_connection = region_resource_id_status.Connection('url') + my_connection = region_resource_id_status.ResStatusConnection('url') my_connection.add_update_status_record('timestamp', 'region', 'status', @@ -91,7 +91,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase): """Test that no exception is raised when calling add_update_status_record. where record does not exist """ - my_connection = region_resource_id_status.Connection('url') + my_connection = region_resource_id_status.ResStatusConnection('url') my_connection.add_update_status_record('timestamp', 'region', 'status', @@ -106,7 +106,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase): return_value=MyFacade(True, False)) def test_add_update_status_record_duplicate_entry(self, mock_engine): """No exception is raised when trying to add a duplicate entry.""" - my_connection = region_resource_id_status.Connection('url') + my_connection = region_resource_id_status.ResStatusConnection('url') my_connection.add_update_status_record('timestamp', 'region', 'status', @@ -118,10 +118,10 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase): 'err_code') @mock.patch.object(region_resource_id_status, 'StatusModel') - @patch.object(region_resource_id_status.Connection, + @patch.object(region_resource_id_status.ResStatusConnection, 'get_timestamp_pair', return_value=(1, 2)) - @mock.patch.object(region_resource_id_status, 'Model') + @mock.patch.object(region_resource_id_status, 'ResourceStatusModel') @mock.patch.object(region_resource_id_status, 'LegacyEngineFacade', return_value=MyFacade(False, False, True)) def test_get_records_by_filter_args_no_records(self, mock_engine, @@ -129,14 +129,14 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase): mock_model, mock_statusmodel): """Test that the function returns None when it got no records.""" - my_connection = region_resource_id_status.Connection('url') + my_connection = region_resource_id_status.ResStatusConnection('url') self.assertIsNone(my_connection.get_records_by_filter_args()) @mock.patch.object(region_resource_id_status, 'StatusModel') - @patch.object(region_resource_id_status.Connection, + @patch.object(region_resource_id_status.ResStatusConnection, 'get_timestamp_pair', return_value=(1, 2)) - @mock.patch.object(region_resource_id_status, 'Model') + @mock.patch.object(region_resource_id_status, 'ResourceStatusModel') @mock.patch.object(region_resource_id_status, 'LegacyEngineFacade', return_value=MyFacade(False, True, True)) def test_get_records_by_filter_args_with_records(self, @@ -145,14 +145,14 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase): mock_model, mock_statusmodel): """Test that the function returns None when it got records.""" - my_connection = region_resource_id_status.Connection('url') + my_connection = region_resource_id_status.ResStatusConnection('url') my_connection.get_records_by_filter_args() @mock.patch.object(region_resource_id_status, 'StatusModel') - @patch.object(region_resource_id_status.Connection, + @patch.object(region_resource_id_status.ResStatusConnection, 'get_timestamp_pair', return_value=(1, 2)) - @mock.patch.object(region_resource_id_status, 'Model') + @mock.patch.object(region_resource_id_status, 'ResourceStatusModel') @mock.patch.object(region_resource_id_status, 'LegacyEngineFacade', return_value=MyFacade(False, False, True)) def test_get_records_by_resource_id_sanity(self, mock_engine, @@ -160,7 +160,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase): mock_model, mock_statusmodel): """No exception is raised when calling get_records_by_resource_id.""" - my_connection = region_resource_id_status.Connection('url') + my_connection = region_resource_id_status.ResStatusConnection('url') my_connection.get_records_by_resource_id('test') @mock.patch.object(region_resource_id_status, 'LegacyEngineFacade', @@ -170,15 +170,15 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase): def test_get_timestamp_pair_sanity(self, mock_engine, time_mock, conf_mock): """Test get_timestamp_pair""" conf_mock.region_resource_id_status.max_interval_time.default = 1 - my_connection = region_resource_id_status.Connection('url') + my_connection = region_resource_id_status.ResStatusConnection('url') (timestamp, ref_timestamp) = my_connection.get_timestamp_pair() self.assertEqual(timestamp, 80000) @mock.patch.object(region_resource_id_status, 'StatusModel') - @patch.object(region_resource_id_status.Connection, + @patch.object(region_resource_id_status.ResStatusConnection, 'get_timestamp_pair', return_value=(1, 2)) - @mock.patch.object(region_resource_id_status, 'Model') + @mock.patch.object(region_resource_id_status, 'ResourceStatusModel') @mock.patch.object(region_resource_id_status, 'LegacyEngineFacade', return_value=MyFacade(False, False, True)) def test_get_records_by_resource_id_and_status_no_records(self, mock_engine, @@ -186,31 +186,31 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase): mock_model, mock_statusmodel): """Test that the function returns None when it got no records.""" - my_connection = region_resource_id_status.Connection('url') + my_connection = region_resource_id_status.ResStatusConnection('url') self.assertIsNone(my_connection.get_records_by_resource_id_and_status('1', '2')) @mock.patch.object(region_resource_id_status, 'StatusModel') - @patch.object(region_resource_id_status.Connection, 'get_timestamp_pair', + @patch.object(region_resource_id_status.ResStatusConnection, 'get_timestamp_pair', return_value=(1, 2)) - @mock.patch.object(region_resource_id_status, 'Model') + @mock.patch.object(region_resource_id_status, 'ResourceStatusModel') @mock.patch.object(region_resource_id_status, 'LegacyEngineFacade', return_value=MyFacade(False, True, True)) def test_get_records_by_resource_id_and_status_sanity(self, mock_engine, mock_get_timestamp, mock_model, mock_statusmodel): - my_connection = region_resource_id_status.Connection('url') + my_connection = region_resource_id_status.ResStatusConnection('url') my_connection.get_records_by_resource_id_and_status('1', '2') @mock.patch.object(region_resource_id_status, 'StatusModel') - @patch.object(region_resource_id_status.Connection, 'get_timestamp_pair', + @patch.object(region_resource_id_status.ResStatusConnection, 'get_timestamp_pair', return_value=(1, 0)) - @mock.patch.object(region_resource_id_status, 'Model') + @mock.patch.object(region_resource_id_status, 'ResourceStatusModel') @mock.patch.object(region_resource_id_status, 'LegacyEngineFacade', return_value=MyFacade(False, True, True)) def test_get_records_by_resource_id_and_status_with_records(self, mock_engine, mock_get_timestamp, mock_model, mock_statusmodel): - my_connection = region_resource_id_status.Connection('url') + my_connection = region_resource_id_status.ResStatusConnection('url') my_connection.get_records_by_resource_id_and_status('1', '2') diff --git a/orm/tests/unit/rds/storage/test_region_resource_id_status.py b/orm/tests/unit/rds/storage/test_region_resource_id_status.py index fb2dab9f..8bc46ee6 100644 --- a/orm/tests/unit/rds/storage/test_region_resource_id_status.py +++ b/orm/tests/unit/rds/storage/test_region_resource_id_status.py @@ -1,7 +1,5 @@ import unittest -from orm.services.resource_distributor.rds.storage.region_resource_id_status import Base - class BaseStorageTests(unittest.TestCase): diff --git a/requirements.txt b/requirements.txt index 2d355af4..2bea0eef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,3 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - pbr!=2.1.0,>=2.0.0 # Apache-2.0 pecan>=1.3.1 # BSD netifaces>=0.10.8 # MIT