x/ranger
Code Issues Proposed changes

Update resource tables layout structure

Browse Source 
Remove  previously-added foreign key in resource_status
table that causes issue with updating resource status table.

Also update resource_template_data table to include region field
and replace transaction_id with resource_id.

Modify RDS logic accordingly.

Change-Id: I71b61b0cfb76545058ef26c0117f007b6d6784d3
This commit is contained in:
st6218 2019-12-18 16:19:23 -08:00
parent b3541d3a49
commit 19b035bfb6
8 changed files with 73 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
orm/services/resource_distributor/rds/ordupdate/ord_notifier.py
View File

@ -159,6 +159,7 @@ def _notify(ord_url,
files=files,
headers=headers,
cert=conf.ordupdate.cert_path)
except requests.exceptions.SSLError:
logger.debug('Received an SSL error (is the certificate valid?)')
raise
@ -195,12 +196,13 @@ def _update_audit(lcp_name, application_id, tracking_id, transaction_id,
# new for template data table
def _update_template_data(transaction_id, resource_name,
def _update_template_data(resource_id, resource_name, region_id,
resource_template_data):
data_to_save = dict(
transaction_id=transaction_id,
stack_name=resource_name,
resource_id=resource_id,
resource_name=resource_name,
region=region_id,
stack_template=resource_template_data.encode("utf-8"))
regionResourceIdStatus.add_update_template_data(data_to_save)
@ -293,7 +295,7 @@ def notify_ord(transaction_id,
raise OrdNotFoundError(Exception(message))
if operation != 'create':
record = regionResourceIdStatus.get_template_by_trans_id(
record = regionResourceIdStatus.get_template_data(
resource_id, region_id)
resource_template_version = record.template_version + 1
@ -336,8 +338,8 @@ def notify_ord(transaction_id,
event_details, status)
if operation in ('create', 'modify'):
# add entry to resource_template_data table
_update_template_data(resource_transaction_id, resource_name,
resource_template_data)
_update_template_data(resource_id, resource_name,
region_id, resource_template_data)
logger.debug(
"Create Resource Requested to ranger agent: region=%s resource_id=%s "

6
orm/services/resource_distributor/rds/services/model/region_resource_id_status.py
View File

@ -47,12 +47,14 @@ class ResourceStatusModel(object):
class ResourceTemplateModel(object):
def __init__(self,
transaction_id,
resource_id,
resource_name,
region,
template_version,
template_data):
self.transaction_id = transaction_id
self.resource_id = resource_id
self.resource_name = resource_name
self.region = region
self.template_version = template_version
self.template_data = template_data

39
orm/services/resource_distributor/rds/services/region_resource_id_status.py
View File

@ -17,13 +17,14 @@ num_of_miliseconds_in_seconds = 1000
def add_update_template_data(data):
logger.debug("update template stack name [{}], and "
"transaction_id [{}] ".format(data['stack_name'],
data['transaction_id']))
logger.debug(
"add/update template stack name [{}] ".format(data['resource_name']))
try:
conn = factory.get_resource_stack_data_connection()
conn.add_update_template_record(data['transaction_id'],
data['stack_name'],
conn.add_update_template_record(data['resource_id'],
data['resource_name'],
data['region'],
data['stack_template'])
except Exception:
@ -36,15 +37,13 @@ def delete_resource_status_data(resource_id):
pass
def get_template_by_trans_id(resource_id, region):
logger.debug("get template transaction id for resource %s and "
def get_template_data(resource_id, region):
logger.debug("get template data for resource %s and "
"region %s" % (resource_id, region))
try:
conn = factory.get_region_resource_id_status_connection()
resource_trans_id = conn.get_resource_region_data(resource_id, region)
conn = factory.get_resource_stack_data_connection()
record = conn.get_resource_template_data(resource_trans_id)
record = conn.get_resource_template_data(resource_id, region)
return record
except Exception:
@ -73,15 +72,13 @@ def add_status(data):
conn = factory.get_region_resource_id_status_connection()
resource_transaction_id = \
conn.add_update_status_record(
data['timestamp'], data['region'], data['status'],
data['transaction_id'], data['resource_id'],
data['ord_notifier_id'], data['error_msg'],
data['error_code'], data['resource_operation'],
data.get('resource_extra_metadata'))
return resource_transaction_id
# post_data_to_image(data)
conn.add_update_status_record(
data['timestamp'], data['region'], data['status'],
data['transaction_id'], data['resource_id'],
data['ord_notifier_id'], data['error_msg'],
data['error_code'], data['resource_operation'],
data.get('resource_extra_metadata'))
except Error as e:
logger.exception("invalid inputs error")
raise
@ -109,9 +106,9 @@ def delete_resource_template_data(resource_id, region):
logger.debug("delete resource_template_data for resource %s and "
"region %s" % (resource_id, region))
conn = factory.get_region_resource_id_status_connection()
resource_trans_id = conn.get_resource_region_data(resource_id, region)
resource_id, region = conn.get_resource_region_data(resource_id, region)
conn = factory.get_resource_stack_data_connection()
conn.delete_resource_template(resource_trans_id)
conn.delete_resource_template(resource_id, region)
def validate_resource_type(resource_type):

46
orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py
View File

@ -68,10 +68,10 @@ class ResourceTemplateRecord(Base):
__tablename__ = 'resource_template_data'
id = Column(Integer, autoincrement=True, primary_key=True)
transaction_id = Column(Text,
ForeignKey('resource_status.transaction_id'),
primary_key=False)
resource_id = Column(Text,
primary_key=False)
resource_name = Column(Text, primary_key=False)
region = Column(Text, primary_key=False)
template_version = Column(Integer, primary_key=False)
template_data = Column(BLOB, primary_key=False)
@ -173,7 +173,7 @@ class ResStatusConnection(region_resource_id_status.ResourceStatusBase):
logger.exception(
'No resource status record with resource id {} and'
'region {} found'.format(resource_id, region))
return record.transaction_id
return record.resource_id, record.region
except Exception as exp:
raise
@ -289,19 +289,20 @@ class ResTemplateConnection(region_resource_id_status.ResourceTemplateBase):
self._engine_facade = LegacyEngineFacade(url)
def add_update_template_record(self,
transaction_id,
resource_id,
resource_name,
region_name,
resource_template):
logger.debug("Add/Update template record:\ntransaction_id [{}]\n"
"resource name [{}]\n\n".format(transaction_id,
resource_name))
logger.debug("Add/Update template record:\nresource_id [{}]\n"
"region [{}]\n\n".format(resource_id, region_name))
try:
session = self._engine_facade.get_session()
with session.begin():
record = session.query(ResourceTemplateRecord).\
filter_by(transaction_id=transaction_id).first()
filter_by(resource_id=resource_id,
region=region_name).first()
if record is not None:
logger.debug("Update resource template record")
record.resource_name = resource_name
@ -311,8 +312,9 @@ class ResTemplateConnection(region_resource_id_status.ResourceTemplateBase):
logger.debug("Add resource template record")
resource_template_record = \
ResourceTemplateRecord(
transaction_id=transaction_id,
resource_id=resource_id,
resource_name=resource_name,
region=region_name,
template_version=0,
template_data=resource_template)
@ -321,33 +323,35 @@ class ResTemplateConnection(region_resource_id_status.ResourceTemplateBase):
except oslo_db.exception.DBDuplicateEntry as e:
logger.warning("Duplicate entry: {}".format(str(e)))
def delete_resource_template(self, trans_id):
def delete_resource_template(self, resource_id, region_name):
try:
session = self._engine_facade.get_session()
with session.begin():
record = session.query(ResourceTemplateRecord).\
filter_by(transaction_id=trans_id).delete()
filter_by(resource_id=resource_id,
region=region_name).delete()
if record is None:
logger.exception(
'Template data not found with transaction'
'id {}'.format(trans_id))
'Template data not found with resource id {} and '
'region name {}'.format(resource_id, region_name))
except Exception as exp:
raise
def get_resource_template_data(self, transaction_id):
logger.debug("Get resource template data by transaction "
"id '{}' ".format(transaction_id))
def get_resource_template_data(self, resource_id, region):
logger.debug("Get resource template data by resource {} and "
"region '{}' ".format(resource_id, region))
try:
session = self._engine_facade.get_session()
with session.begin():
record = session.query(ResourceTemplateRecord).\
filter_by(transaction_id=transaction_id).first()
filter_by(resource_id=resource_id,
region=region).first()
if record is None:
logger.exception(
'No resource template record found with transaction '
'id '.format(transaction_id))
'No template record found with resource id {} '
'and region {} '.format(resource_id, region))
return record
except Exception as exp:
raise

6
orm/services/resource_distributor/rds/storage/region_resource_id_status.py
View File

@ -29,9 +29,9 @@ class ResourceTemplateBase(object):
pass
def add_update_template_record(self,
transaction_id,
stack_name,
stacK_template):
resource_id,
stack_name, region,
stack_template):
raise NotImplementedError(Exception("Please Implement this method"))
def get_records_by_transaction_id(self, transaction_id):

3
orm/services/resource_distributor/rds/utils/utils.py
View File

@ -28,6 +28,9 @@ def post_data_to_image(data):
def invoke_delete_region(data):
# ALLOW delete region for the following conditions:
# (1) if 'delete' operation success
# (2) if error = 'ORD_12' (heat stack not found)
if data['resource_operation'] == 'delete' and (data['status'] == 'Success' or data['error_code'] == 'ORD_012'):
rds_resource_service_proxy.invoke_resources_region_delete(
resource_type=data['resource_type'],

13
orm/services/resource_distributor/scripts/db_scripts/create_db.sql
View File

@ -20,8 +20,8 @@ create table if not exists resource_status
err_code varchar(64),
operation varchar(64),
primary key (id),
unique(resource_id, region),
unique transaction_idx (transaction_id));
unique(resource_id, region)
);
#*****
@ -31,12 +31,13 @@ create table if not exists resource_status
create table if not exists resource_template_data
(
id integer auto_increment not null,
transaction_id varchar(64),
resource_name varchar(64) NOT NULL,
template_version integer,
resource_id varchar(64) not null,
resource_name varchar(64) not null,
region varchar(64) not null,
template_version integer not null,
template_data BLOB NOT NULL,
primary key (id),
foreign key (transaction_id) references resource_status(transaction_id) ON DELETE CASCADE
foreign key (resource_id, region) references resource_status(resource_id, region) ON DELETE CASCADE
);

9
orm/tests/unit/rds/services/model/test_region_resource_id_status.py
View File

@ -26,13 +26,14 @@ class TestModel(unittest.TestCase):
class TestResourceTemplateModel(unittest.TestCase):
def test_model_as_dict(self):
model = region_resource_id_status.ResourceTemplateModel(1, 2, 3, 4)
model = region_resource_id_status.ResourceTemplateModel(1, 2, 3, 4, 5)
expected_template_dict = {
'transaction_id': 1,
'resource_id': 1,
'resource_name': 2,
'template_version': 3,
'template_data': 4
'region': 3,
'template_version': 4,
'template_data': 5
}
test_dict = model.as_dict()