x/ranger
Code Issues Proposed changes

Pep8 and Ranger Image Build process update

Browse Source 
Updated pep8 requirements and fixed resulting
codes to fix broken pep8 job.

Updated docker image build process to use roles
to ensure docker rather than using manual docker
install process to fix broken ranger image job.

Co-Authored-By: Jeremy Houser <jeremyhouser@protonmail.com>
Co-Authored-By: Chi Lo <cl566n@att.com>

Change-Id: I28df0a27e4b354dd53c17fbb1a9468cb7ff5bc16
This commit is contained in:
Gage Hugo 2020-09-23 11:32:58 -05:00
parent 8366b10bef
commit 68a500b3de
44 changed files with 287 additions and 307 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
orm/base_config.py
View File

@ -215,8 +215,8 @@ else:
series_metadata = [s for s in sections if "_flavor_series_metadata" in s]
for metadata in series_metadata:
flavor = metadata.replace('_flavor_series_metadata', '')
flavor_dict = dict(set(config.items(metadata)) -
set(config.items('DEFAULT')))
flavor_dict = dict(set(config.items(metadata))
- set(config.items('DEFAULT')))
FlavorGroup = [
cfg.DictOpt(flavor,
@ -349,8 +349,8 @@ conn = CONF.database.connection
db_connect = conn.replace("mysql+pymysql", "mysql") if conn else None
ssl_verify = CONF.ssl_verify
token_auth_version = '3' if (CONF.keystone_authtoken.auth_version ==
'v3') else '2.0'
token_auth_version = '3' if (CONF.keystone_authtoken.auth_version
== 'v3') else '2.0'
cert_path = CONF.ranger_agent_client_cert_path
https_enabled = CONF.ranger_agent_https_enabled

4
orm/common/orm_common/model/models.py
View File

@ -50,8 +50,8 @@ class StatusModel(object):
def _get_aggregated_status(self):
is_pending = False
for region in self.regions:
if (region.status == 'Error' and
region.operation.strip() != 'delete'):
if (region.status == 'Error'
and region.operation.strip() != 'delete'):
# If a region had an error, the aggregated status is 'Error'
return 'Error'
elif region.status == 'Submitted':

4
orm/common/orm_common/policy/_parser.py
View File

@ -103,8 +103,8 @@ class ParseState(object):
"""
for reduction, methname in self.reducers:
if (len(self.tokens) >= len(reduction) and
self.tokens[-len(reduction):] == reduction):
if (len(self.tokens) >= len(reduction)
and self.tokens[-len(reduction):] == reduction):
# Get the reduction method
meth = getattr(self, methname)

4
orm/common/orm_common/sql_alchemy/resource_status_record.py
View File

@ -54,8 +54,8 @@ class ResourceStatusRecord:
if records:
for id, timestamp, region, status, transaction, resource, \
notifier, err_msg, err_code, operation in records:
if (status == "Submitted" and
timestamp < ref_timestamp):
if (status == "Submitted"
and timestamp < ref_timestamp):
timestamp = timestamp
status = "Error"
err_msg = "Status updated to 'Error'. " \

1
orm/common/orm_common/utils/api_error_utils.py
View File

@ -34,6 +34,7 @@ def get_error_dict(status_code, transaction_id, message, error_details=""):
'details': error_details
}
# Default error messages
error_message = {
400: {'message': 'Incompatible JSON body', 'type': 'Bad Request'},

1
orm/orm_client/flavorgen/make_flav.py
View File

@ -19,6 +19,7 @@ def calculate_name(flavor):
return "{0}.c{1}r{2}d{3}".format(flavor['series'], flavor['vcpus'],
flavor['ram'], flavor['disk'])
data = {
"series": "gv",
"vcpus": "10",

1
orm/services/customer_manager/cms_rest/app.py
View File

@ -43,5 +43,6 @@ def main(argv=None):
runner = CommandRunner()
runner.run(['serve', path + '/config.py'])
if __name__ == "__main__":
main()

6
orm/services/customer_manager/cms_rest/controllers/v1/orm/customer/enabled.py
View File

@ -44,12 +44,12 @@ class EnabledController(rest.RestController):
@wsexpose(None, str, rest_content_types='json')
def post(self, customer_id):
raise err_utils.get_error(request.transaction_id, status_code=405)
raise err_utils.get_error(request.transaction_id, status_code=405)
@wsexpose(None, str, rest_content_types='json')
def get(self, customer_id):
raise err_utils.get_error(request.transaction_id, status_code=405)
raise err_utils.get_error(request.transaction_id, status_code=405)
@wsexpose(None, str, rest_content_types='json')
def delete(self, customer_id):
raise err_utils.get_error(request.transaction_id, status_code=405)
raise err_utils.get_error(request.transaction_id, status_code=405)

8
orm/services/customer_manager/cms_rest/controllers/v1/orm/customer/metadata.py
View File

@ -40,10 +40,10 @@ class MetadataController(rest.RestController):
message=str(ex), status_code=404)
except ErrorStatus as ex:
LOG.log_exception("MetaDataController - Failed to add metadata", ex)
raise err_utils.get_error(request.transaction_id,
message=ex.message,
status_code=ex.status_code)
LOG.log_exception("MetaDataController - Failed to add metadata", ex)
raise err_utils.get_error(request.transaction_id,
message=ex.message,
status_code=ex.status_code)
except LookupError as ex:
LOG.log_exception("MetaDataController - {0}".format(ex.message), ex)

4
orm/services/customer_manager/cms_rest/controllers/v1/orm/group/region_users.py
View File

@ -19,8 +19,8 @@ class RegionUserController(rest.RestController):
@wsexpose([str], str, str, str, rest_content_types='json')
def get(self, group_id, region_id, user_id):
return ["This is groups region user controller for group id: " +
group_id]
return ["This is groups region user controller for group id: "
+ group_id]
@wsexpose(RegionUserResultWrapper, str, str, body=[RegionUser],
rest_content_types='json', status_code=200)

16
orm/services/customer_manager/cms_rest/data/data_manager.py
View File

@ -143,8 +143,8 @@ class DataManager(object):
self.group_record = GroupRecord(self.session)
return self.group_record
elif (record_name == "CustomerRegion" or
record_name == "customer_region"):
elif (record_name == "CustomerRegion"
or record_name == "customer_region"):
if not hasattr(self, "customer_region_record"):
self.customer_region_record = CustomerRegionRecord(
self.session)
@ -166,22 +166,22 @@ class DataManager(object):
self.groups_role_record = GroupsRoleRecord(self.session)
return self.groups_role_record
elif (record_name == "GroupsCustomerRole" or
record_name == "groups_customer_role"):
elif (record_name == "GroupsCustomerRole"
or record_name == "groups_customer_role"):
if not hasattr(self, "groups_customer_role_record"):
self.groups_customer_role_record = GroupsCustomerRoleRecord(
self.session)
return self.groups_customer_role_record
elif (record_name == "GroupsDomainRole" or
record_name == "groups_domain_role"):
elif (record_name == "GroupsDomainRole"
or record_name == "groups_domain_role"):
if not hasattr(self, "groups_domain_role_record"):
self.groups_domain_role_record = GroupsDomainRoleRecord(
self.session)
return self.groups_domain_role_record
elif (record_name == "GroupsUser" or
record_name == "groups_user"):
elif (record_name == "GroupsUser"
or record_name == "groups_user"):
if not hasattr(self, "groups_user_record"):
self.groups_user_record = GroupsUserRecord(
self.session)

4
orm/services/customer_manager/cms_rest/data/sql_alchemy/groups_customer_role_record.py
View File

@ -36,8 +36,8 @@ class GroupsCustomerRoleRecord:
self.session.add(groups_customer_role)
except Exception as exception:
LOG.log_exception(
"Failed to insert groups_customer_role" +
str(groups_customer_role), exception)
"Failed to insert groups_customer_role"
+ str(groups_customer_role), exception)
raise
def get_customer_roles_by_region(self,

4
orm/services/customer_manager/cms_rest/data/sql_alchemy/groups_domain_role_record.py
View File

@ -36,8 +36,8 @@ class GroupsDomainRoleRecord:
self.session.add(groups_domain_role)
except Exception as exception:
LOG.log_exception(
"Failed to insert groups_domain_role" +
str(groups_domain_role), exception)
"Failed to insert groups_domain_role"
+ str(groups_domain_role), exception)
raise
def get_domain_roles_by_region(self,

4
orm/services/customer_manager/cms_rest/data/sql_alchemy/models.py
View File

@ -243,8 +243,8 @@ class GroupsRegion(Base, CMSBaseModel):
def get_group_users(self):
group_users = []
for group_user in self.groups_users:
if (group_user.region_id == self.region_id and
group_user.group_id == self.group_id):
if (group_user.region_id == self.region_id
and group_user.group_id == self.group_id):
group_users.append(group_user)
return group_users

5
orm/services/customer_manager/cms_rest/data/sql_alchemy/user_role_record.py
View File

@ -1,6 +1,6 @@
from orm.services.customer_manager.cms_rest.data.sql_alchemy.cms_user_record import CmsUserRecord
from orm.services.customer_manager.cms_rest.data.sql_alchemy.customer_record import CustomerRecord
from orm.services.customer_manager.cms_rest.data.sql_alchemy.models import *
from orm.services.customer_manager.cms_rest.data.sql_alchemy.models import UserRole
from orm.services.customer_manager.cms_rest.data.sql_alchemy.region_record import RegionRecord
from orm.services.customer_manager.cms_rest.logger import get_logger
from orm.services.customer_manager.cms_rest.logic.error_base import NotFound
@ -12,7 +12,8 @@ class UserRoleRecord:
def __init__(self, session=None):
# this model uses for the parameters for any access methods - not as instance of record in the table
# this model uses for the parameters for any access methods,
# not as instance of record in the table
self.__user_role = UserRole()
# self.setRecordData(self.user_role)
# self.user_role.Clear()

1
orm/services/customer_manager/cms_rest/logger/__init__.py
View File

@ -7,4 +7,5 @@ def get_logger(name):
return logger
__all__ = ['get_logger']

28
orm/services/customer_manager/cms_rest/logic/group_logic.py
View File

@ -91,9 +91,9 @@ class GroupLogic(object):
except Exception as ex:
if hasattr(ex, 'orig') and ex.orig[0] == 1062:
raise DuplicateEntryError(
'Error, duplicate entry, region ' +
region.name +
' already associated with group')
'Error, duplicate entry, region '
+ region.name
+ ' already associated with group')
raise ex
self.add_user_db(datamanager, region.users,
@ -112,8 +112,8 @@ class GroupLogic(object):
default_user_exists =\
[username for exist_user in existing_default_users
if exist_user.user.name == username and
exist_user.domain_name == domain_value]
if exist_user.user.name == username
and exist_user.domain_name == domain_value]
if not default_user_exists:
# add user to cms_user table and group_users
@ -135,8 +135,8 @@ class GroupLogic(object):
# in existing_default_users list
region_user_exists =\
[username for exist_user in all_existing_users
if exist_user.user.name == username and
exist_user.domain_name == domain_value]
if exist_user.user.name == username
and exist_user.domain_name == domain_value]
if not region_user_exists:
# add user to cms_user table and group_users
@ -508,8 +508,8 @@ class GroupLogic(object):
group_uuid, region_id, customer_id, role_id)
if (not customer.check_groups_customer_role_exist(
role_id, group_uuid) and
not domain.check_groups_domain_role_exist(
role_id, group_uuid)
and not domain.check_groups_domain_role_exist(
role_id, group_uuid)):
groups_role.remove_role_from_group(group_uuid, role_id)
@ -545,8 +545,8 @@ class GroupLogic(object):
RdsProxy.send_group_dict(group_dict, transaction_id, "POST")
else:
LOG.debug(
"Group with no regions - wasn't send to RDS Proxy " +
str(group))
"Group with no regions - wasn't send to RDS Proxy "
+ str(group))
datamanager.commit()
@ -718,9 +718,9 @@ class GroupLogic(object):
ret_group = sql_group.to_wsme()
if sql_group.get_group_regions():
resp = requests.get(conf.api.rds_server.base +
conf.api.rds_server.status +
sql_group.uuid, verify=conf.verify).json()
resp = requests.get(conf.api.rds_server.base
+ conf.api.rds_server.status
+ sql_group.uuid, verify=conf.verify).json()
for item in ret_group.regions:
for status in resp['regions']:
if status['region'] == item.name:

32
orm/services/customer_manager/cms_rest/rds_proxy.py
View File

@ -16,17 +16,17 @@ class RdsProxy(object):
@staticmethod
def get_status(resource_id):
try:
LOG.debug(
"Sending to RDS Server to get status: " +
conf.api.rds_server.base + conf.api.rds_server.status +
resource_id)
resp = requests.get(
conf.api.rds_server.base + conf.api.rds_server.status +
resource_id, verify=conf.verify)
LOG.debug(
"Sending to RDS Server to get status: " +
conf.api.rds_server.base +
conf.api.rds_server.status + resource_id)
LOG.debug("Sending to RDS Server to get status: "
+ conf.api.rds_server.base
+ conf.api.rds_server.status
+ resource_id)
resp = requests.get(conf.api.rds_server.base
+ conf.api.rds_server.status
+ resource_id, verify=conf.verify)
LOG.debug("Sending to RDS Server to get status: "
+ conf.api.rds_server.base
+ conf.api.rds_server.status
+ resource_id)
pp = pprint.PrettyPrinter(width=30)
pretty_text = pp.pformat(resp.json())
LOG.debug("Response from RDS Server:\n" + pretty_text)
@ -95,8 +95,9 @@ class RdsProxy(object):
LOG.debug("Wrapper JSON before sending action: {0} to Rds "
"Proxy\n{1}".format(method, pretty_text))
LOG.info("Sending to RDS Server: " +
conf.api.rds_server.base + conf.api.rds_server.resources)
LOG.info("Sending to RDS Server: "
+ conf.api.rds_server.base
+ conf.api.rds_server.resources)
wrapper_json = json.dumps(data)
@ -175,8 +176,9 @@ class RdsProxy(object):
LOG.debug("Wrapper JSON before sending action: {0} to Rds "
"Proxy\n{1}".format(method, pretty_text))
LOG.info("Sending to RDS Server: " + conf.api.rds_server.base +
conf.api.rds_server.resources)
LOG.info("Sending to RDS Server: "
+ conf.api.rds_server.base
+ conf.api.rds_server.resources)
wrapper_json = json.dumps(data)

18
orm/services/db_setup.py
View File

@ -16,7 +16,7 @@
from configparser import ConfigParser
from oslo_config import cfg
import re
from sqlalchemy import *
from sqlalchemy import create_engine
import sys
CONF = cfg.CONF
@ -106,14 +106,14 @@ def main(argv=None):
CONF.ranger_base + '/orm/services/id_generator/scripts/db_scripts/db_create.sql',
CONF.ranger_base + '/orm/services/resource_distributor/scripts/db_scripts/create_db.sql',
CONF.ranger_base + '/orm/services/region_manager/scripts/db_scripts/create_db.sql',
CONF.ranger_base +
'/orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql',
CONF.ranger_base +
'/orm/services/customer_manager/scripts/db_scripts/ranger_cms_update_db.sql',
CONF.ranger_base +
'/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql',
CONF.ranger_base +
'/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_update_db.sql',
CONF.ranger_base
+ '/orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql',
CONF.ranger_base
+ '/orm/services/customer_manager/scripts/db_scripts/ranger_cms_update_db.sql',
CONF.ranger_base
+ '/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql',
CONF.ranger_base
+ '/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_update_db.sql',
CONF.ranger_base + '/orm/services/image_manager/scripts/db_scripts/create_db.sql'
]

4
orm/services/flavor_manager/fms_rest/data/sql_alchemy/flavor_extra_spec/flavor_extra_spec_record.py
View File

@ -34,8 +34,8 @@ class FlavorExtraSpecRecord:
try:
self.session.add(flavor_extra_specs)
except Exception as exception:
LOG.log_exception("Failed to insert FlavorExtraSpec" +
str(flavor_extra_specs), exception)
LOG.log_exception("Failed to insert FlavorExtraSpec"
+ str(flavor_extra_specs), exception)
raise
def get_flavor_extra_spec(self, flavor_internal_id, key_name):

4
orm/services/flavor_manager/fms_rest/data/sql_alchemy/flavor_options/flavor_option_record.py
View File

@ -33,8 +33,8 @@ class FlavorOptionRecord:
try:
self.session.add(flavor_option)
except Exception as exception:
LOG.log_exception("Failed to insert FlavorOption" +
str(flavor_option), exception)
LOG.log_exception("Failed to insert FlavorOption"
+ str(flavor_option), exception)
raise
def get_flavor_option(self, flavor_internal_id, key_name):

4
orm/services/flavor_manager/fms_rest/data/sql_alchemy/flavor_tags/flavor_tag_record.py
View File

@ -33,8 +33,8 @@ class FlavorTagRecord:
try:
self.session.add(flavor_tag)
except Exception as exception:
LOG.log_exception("Failed to insert FlavorTag" +
str(flavor_tag), exception)
LOG.log_exception("Failed to insert FlavorTag"
+ str(flavor_tag), exception)
raise
def get_flavor_extra_spec(self, flavor_internal_id, key_name):

23
orm/services/flavor_manager/fms_rest/data/wsme/models.py
View File

@ -271,8 +271,8 @@ class Flavor(Model):
# validate series and set flavor vcpu and vram limits
requested_numa = [n for n in valid_numa if n in
list(self.options.keys()) and
self.options[n].lower() == 'true']
list(self.options.keys())
and self.options[n].lower() == 'true']
if requested_numa:
vcpu_limit = int(series_metadata['vcpu_limit'])
@ -314,8 +314,9 @@ class Flavor(Model):
if not self.ephemeral:
self.ephemeral = "0"
elif (self.ephemeral and
int(self.ephemeral) not in list(range(0, ephemeral_limit + 1))):
elif (self.ephemeral
and int(self.ephemeral) not in
list(range(0, ephemeral_limit + 1))):
raise ErrorStatus(400,
"ephemeral value is out of range. Expected"
" range is 0-%5d(%2dTB)" %
@ -468,8 +469,8 @@ class Flavor(Model):
series_metadata['valid_options_numa'].split(',')]
option_numa = [n for n in valid_numa if n in
list(self.options.keys()) and
self.options[n].lower() == 'true']
list(self.options.keys())
and self.options[n].lower() == 'true']
if not option_numa:
es = db_models.FlavorExtraSpec(
@ -487,8 +488,8 @@ class Flavor(Model):
series_metadata['valid_options_pci'].split(',')]
option_pci = [n for n in valid_pci if n in
list(self.options.keys()) and
self.options[n].lower() == 'true']
list(self.options.keys())
and self.options[n].lower() == 'true']
if option_pci:
es = db_models.FlavorExtraSpec(
@ -503,9 +504,9 @@ class Flavor(Model):
series_metadata['valid_options_thread'].split(',')]
option_thread = [n for n in valid_thread if n in
list(self.options.keys()) and
self.options[n].lower() == 'true' and
self.visibility.lower() == 'private']
list(self.options.keys())
and self.options[n].lower() == 'true'
and self.visibility.lower() == 'private']
if option_thread:
es = db_models.FlavorExtraSpec(

1
orm/services/flavor_manager/fms_rest/logger/__init__.py
View File

@ -7,4 +7,5 @@ def get_logger(name):
return logger
__all__ = ['get_logger']

8
orm/services/flavor_manager/fms_rest/logic/flavor_logic.py
View File

@ -134,8 +134,8 @@ def send_to_rds_if_needed(sql_flavor,
LOG.debug("Flavor is valid to send to RDS - sending to RDS Proxy ")
rds_proxy.send_flavor(flavor_dict, transaction_id, http_action)
else:
LOG.debug("Flavor with no regions - wasn't sent to RDS Proxy " +
str(sql_flavor))
LOG.debug("Flavor with no regions - wasn't sent to RDS Proxy "
+ str(sql_flavor))
@di.dependsOn('data_manager')
@ -1054,8 +1054,8 @@ def calculate_name(flavor):
series_metadata if x.startswith("valid_options_")]
options = [n for n in valid_options if n in
list(flavor.flavor.options.keys()) and
flavor.flavor.options[n].lower() == 'true']
list(flavor.flavor.options.keys())
and flavor.flavor.options[n].lower() == 'true']
if 't0' in options and flavor.flavor.visibility.lower() != 'private':
options.remove('t0')

130
orm/services/flavor_manager/fms_rest/proxies/rds_proxy.py
View File

@ -16,81 +16,79 @@ headers = {'content-type': 'application/json'}
def send_flavor(flavor_dict, transaction_id, action="put"):
# action can be "post" for creating flavor or "delete" for deleting flavor
# action can be "post" for creating flavor or "delete" for deleting flavor
data = {
"service_template":
{
"resource": {
"resource_type": "flavor"
},
"model": str(json.dumps(flavor_dict)),
"tracking": {
"external_id": "",
"tracking_id": transaction_id
}
}
data = {
"service_template": {
"resource": {
"resource_type": "flavor"
},
"model": str(json.dumps(flavor_dict)),
"tracking": {
"external_id": "",
"tracking_id": transaction_id
}
}
}
data_to_display = {
"service_template":
{
"resource": {
"resource_type": "flavor"
},
"model": flavor_dict,
"tracking": {
"external_id": "",
"tracking_id": transaction_id
}
}
data_to_display = {
"service_template": {
"resource": {
"resource_type": "flavor"
},
"model": flavor_dict,
"tracking": {
"external_id": "",
"tracking_id": transaction_id
}
}
}
pp = pprint.PrettyPrinter(width=30)
pretty_text = pp.pformat(data_to_display)
wrapper_json = json.dumps(data)
pp = pprint.PrettyPrinter(width=30)
pretty_text = pp.pformat(data_to_display)
wrapper_json = json.dumps(data)
headers['X-RANGER-Client'] = request.headers[
'X-RANGER-Client'] if 'X-RANGER-Client' in request.headers else \
'NA'
headers['X-RANGER-Requester'] = request.headers[
'X-RANGER-Requester'] if 'X-RANGER-Requester' in request.headers else \
''
headers['X-Auth-Region'] = request.headers[
'X-Auth-Region'] if 'X-Auth-Region' in \
request.headers else ''
headers['X-Auth-Token'] = request.headers[
'X-Auth-Token'] if 'X-Auth-Token' in \
request.headers else ''
headers['X-RANGER-Client'] = request.headers[
'X-RANGER-Client'] if 'X-RANGER-Client' in request.headers else \
'NA'
headers['X-RANGER-Requester'] = request.headers[
'X-RANGER-Requester'] if 'X-RANGER-Requester' in request.headers else \
''
headers['X-Auth-Region'] = request.headers[
'X-Auth-Region'] if 'X-Auth-Region' in \
request.headers else ''
headers['X-Auth-Token'] = request.headers[
'X-Auth-Token'] if 'X-Auth-Token' in \
request.headers else ''
LOG.debug("Wrapper JSON before sending action: {0} to Rds Proxy {1}".format(action, pretty_text))
LOG.info("Sending to RDS Server: " + conf.api.rds_server.base + conf.api.rds_server.resources)
if action == "post":
resp = requests.post(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers,
verify=conf.verify)
elif action == "put":
resp = requests.put(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers,
verify=conf.verify)
elif action == "delete":
resp = requests.delete(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers,
verify=conf.verify)
else:
raise Exception("Invalid action in RdxProxy.send_flavor(flavor_dict, transaction_id, action) action can be post or delete, got {0}".format(action))
LOG.debug("Wrapper JSON before sending action: {0} to Rds Proxy {1}".format(action, pretty_text))
LOG.info("Sending to RDS Server: " + conf.api.rds_server.base + conf.api.rds_server.resources)
if action == "post":
resp = requests.post(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers,
verify=conf.verify)
elif action == "put":
resp = requests.put(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers,
verify=conf.verify)
elif action == "delete":
resp = requests.delete(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers,
verify=conf.verify)
else:
raise Exception("Invalid action in RdxProxy.send_flavor(flavor_dict, transaction_id, action) action can be post or delete, got {0}".format(action))
content = resp.content
LOG.debug("return from rds server status code: {0} content: {1}".format(resp.status_code, resp.content))
if resp.content and 200 <= resp.status_code < 300:
content = resp.json()
else:
raise ErrorStatus(resp.status_code, "Got error from rds server, code: {0} message: {1}".format(resp.status_code, content))
content = resp.content
LOG.debug("return from rds server status code: {0} content: {1}".format(resp.status_code, resp.content))
if resp.content and 200 <= resp.status_code < 300:
content = resp.json()
else:
raise ErrorStatus(resp.status_code, "Got error from rds server, code: {0} message: {1}".format(resp.status_code, content))
return content
return content
def get_status(resource_id):

1
orm/services/flavor_manager/fms_rest/utils/utils.py
View File

@ -88,6 +88,7 @@ def make_transid():
else:
return None
audit_setup = False

1
orm/services/image_manager/ims/logger/__init__.py
View File

@ -7,4 +7,5 @@ def get_logger(name):
return logger
__all__ = ['get_logger']

22
orm/services/image_manager/ims/logic/image_logic.py
View File

@ -426,20 +426,20 @@ def delete_customer(image_uuid, customer_id, transaction_id):
def set_resource_status(sql_image, status_model):
image_wrapper = ImageWrapper.from_db_model(sql_image)
image_wrapper.image.status = 'no regions'
image_wrapper = ImageWrapper.from_db_model(sql_image)
image_wrapper.image.status = 'no regions'
if status_model and status_model.regions:
for region in image_wrapper.image.regions:
for status in status_model.regions:
if status.region == region.name:
region.status = status.status
if status.error_msg:
region.set_error_message(status.error_msg)
if status_model and status_model.regions:
for region in image_wrapper.image.regions:
for status in status_model.regions:
if status.region == region.name:
region.status = status.status
if status.error_msg:
region.set_error_message(status.error_msg)
image_wrapper.image.status = status_model.status
image_wrapper.image.status = status_model.status
return image_wrapper
return image_wrapper
@di.dependsOn('data_manager')

4
orm/services/image_manager/ims/persistency/sql_alchemy/db_models.py
View File

@ -185,7 +185,7 @@ class Image(Base, IMSBaseModel):
try:
LOG.debug("remove regions from image {0}".format(str(self)))
for region in reversed(self.regions):
self.regions.remove(region)
self.regions.remove(region)
except Exception as exception:
LOG.log_exception("Failed to remove regions from image {0}".format(str(self)), exception)
@ -220,7 +220,7 @@ class Image(Base, IMSBaseModel):
LOG.debug("remove customers from image {0}".format(str(self)))
for customer in reversed(self.customers):
self.customers.remove(customer)
self.customers.remove(customer)
except Exception as exception:
LOG.log_exception("Failed to remove customers from image"

147
orm/services/image_manager/ims/proxies/rds_proxy.py
View File

@ -16,89 +16,88 @@ headers = {'content-type': 'application/json'}
def send_image(image_dict, transaction_id, action="put"):
# action can be "post" for creating image or "delete" for deleting image
# action can be "post" for creating image or "delete" for deleting image
data = {
"service_template":
{
"resource": {
"resource_type": "image"
},
"model": str(json.dumps(image_dict)),
"tracking": {
"external_id": "",
"tracking_id": transaction_id
}
}
data = {
"service_template": {
"resource": {
"resource_type": "image"
},
"model": str(json.dumps(image_dict)),
"tracking": {
"external_id": "",
"tracking_id": transaction_id
}
}
}
data_to_display = {
"service_template":
{
"resource": {
"resource_type": "image"
},
"model": image_dict,
"tracking": {
"external_id": "",
"tracking_id": transaction_id
}
}
data_to_display = {
"service_template": {
"resource": {
"resource_type": "image"
},
"model": image_dict,
"tracking": {
"external_id": "",
"tracking_id": transaction_id
}
}
try:
pp = pprint.PrettyPrinter(width=30)
pretty_text = pp.pformat(data_to_display)
wrapper_json = json.dumps(data)
}
headers['X-RANGER-Client'] = request.headers[
'X-RANGER-Client'] if 'X-RANGER-Client' in request.headers else \
'NA'
headers['X-RANGER-Requester'] = request.headers[
'X-RANGER-Requester'] if 'X-RANGER-Requester' in request.headers else \
''
headers['X-Auth-Region'] = request.headers[
'X-Auth-Region'] if 'X-Auth-Region' in \
request.headers else ''
headers['X-Auth-Token'] = request.headers[
'X-Auth-Token'] if 'X-Auth-Token' in \
request.headers else ''
try:
pp = pprint.PrettyPrinter(width=30)
pretty_text = pp.pformat(data_to_display)
wrapper_json = json.dumps(data)
LOG.debug("Wrapper JSON before sending action: {0} to Rds Proxy {1}".format(action, pretty_text))
LOG.info("Sending to RDS Server: " + conf.api.rds_server.base + conf.api.rds_server.resources)
if action == "post":
resp = requests.post(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers, verify=conf.verify)
elif action == "put":
resp = requests.put(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers, verify=conf.verify)
elif action == "delete":
resp = requests.delete(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers, verify=conf.verify)
else:
raise Exception("Invalid action in RdxProxy.send_image("
"image_dict, transaction_id, action) "
"action can be post or delete, "
"got {0}".format(action))
headers['X-RANGER-Client'] = request.headers[
'X-RANGER-Client'] if 'X-RANGER-Client' in request.headers else \
'NA'
headers['X-RANGER-Requester'] = request.headers[
'X-RANGER-Requester'] if 'X-RANGER-Requester' in request.headers else \
''
headers['X-Auth-Region'] = request.headers[
'X-Auth-Region'] if 'X-Auth-Region' in \
request.headers else ''
headers['X-Auth-Token'] = request.headers[
'X-Auth-Token'] if 'X-Auth-Token' in \
request.headers else ''
content = resp.content
LOG.debug("return from rds server status code: {0} "
"content: {1}".format(resp.status_code, resp.content))
if resp.content and 200 <= resp.status_code < 300:
content = resp.json()
else:
# In case of error from rds, the response is WSME format response.
# the error message is within the 'faultstring'
raise ErrorStatus(resp.status_code,
json.loads(content)["faultstring"])
LOG.debug("Wrapper JSON before sending action: {0} to Rds Proxy {1}".format(action, pretty_text))
LOG.info("Sending to RDS Server: " + conf.api.rds_server.base + conf.api.rds_server.resources)
if action == "post":
resp = requests.post(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers, verify=conf.verify)
elif action == "put":
resp = requests.put(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers, verify=conf.verify)
elif action == "delete":
resp = requests.delete(conf.api.rds_server.base + conf.api.rds_server.resources,
data=wrapper_json,
headers=headers, verify=conf.verify)
else:
raise Exception("Invalid action in RdxProxy.send_image("
"image_dict, transaction_id, action) "
"action can be post or delete, "
"got {0}".format(action))
except Exception as exp:
LOG.log_exception("ImageLogic - Failed to update image", exp)
raise exp
content = resp.content
LOG.debug("return from rds server status code: {0} "
"content: {1}".format(resp.status_code, resp.content))
if resp.content and 200 <= resp.status_code < 300:
content = resp.json()
else:
# In case of error from rds, the response is WSME format response.
# the error message is within the 'faultstring'
raise ErrorStatus(resp.status_code,
json.loads(content)["faultstring"])
return content
except Exception as exp:
LOG.log_exception("ImageLogic - Failed to update image", exp)
raise exp
return content
def get_status(resource_id, json_convert=True):

1
orm/services/region_manager/csv2db.py
View File

@ -53,5 +53,6 @@ def main():
data_manager = DataManager(db_url, 3, 3)
load_csv2db(data_manager)
if __name__ == "__main__":
main()

1
orm/services/region_manager/revert_csv2db.py
View File

@ -27,5 +27,6 @@ def main():
data_manager = DataManager(db_url)
revert_csv2db(data_manager)
if __name__ == "__main__":
main()

1
orm/services/region_manager/rms/logger/__init__.py
View File

@ -7,4 +7,5 @@ def get_logger(name):
return logger
__all__ = ['get_logger']

4
orm/services/resource_distributor/rds/services/resource.py
View File

@ -143,8 +143,8 @@ def _create_template_data(input_data):
tenants_list, target['name'])
for tenant in valid_tenants_list:
ok_tenants['tenant_id'] = tenant
ok_tenants_list.append(ok_tenants.copy())
ok_tenants['tenant_id'] = tenant
ok_tenants_list.append(ok_tenants.copy())
# Note: If ok_tenant_list is empty, just create heat template
# for private flavor with empty tenant list

8
orm/tests/functional/uuidgen/test_functional.py
View File

@ -37,7 +37,7 @@ class TestRootController(FunctionalTest):
resp = self.app.post('/v1/uuids')
self.assertEqual(resp.status_int, 200)
body = json.loads(resp.body)
self.assertTrue(re.search('^[\dTZ.:-]+$', body['issued_at']))
self.assertTrue(re.search('^[\\dTZ.:-]+$', body['issued_at']))
self.assertEqual('', body['uuid_type'])
self.assertTrue(re.search('^[0-9a-f]+$', body['uuid']))
@ -47,7 +47,7 @@ class TestRootController(FunctionalTest):
params={'uuid_type': 'milkshake'})
self.assertEqual(resp.status_int, 200)
body = json.loads(resp.body)
self.assertTrue(re.search('^[\dTZ.:-]+$', body['issued_at']))
self.assertTrue(re.search('^[\\dTZ.:-]+$', body['issued_at']))
self.assertEqual('milkshake', body['uuid_type'])
# Same format as with no parameter
self.assertTrue(re.search('^[0-9a-f-]+$', body['uuid']))
@ -56,7 +56,7 @@ class TestRootController(FunctionalTest):
params={'uuid': 1337, 'uuid_type': 'milkshake'})
self.assertEqual(resp.status_int, 200)
body = json.loads(resp.body)
self.assertTrue(re.search('^[\dTZ.:-]+$', body['issued_at']))
self.assertTrue(re.search('^[\\dTZ.:-]+$', body['issued_at']))
self.assertEqual('custId', body['uuid_type'])
# Same format as with no parameter
self.assertTrue(re.search('^[0-9a-f-]+$', body['uuid']))
@ -98,7 +98,7 @@ class TestRootController(FunctionalTest):
expect_errors=True)
self.assertEqual(resp.status_int, 200)
body = json.loads(resp.body)
self.assertTrue(re.search('^[\dTZ.:-]+$', body['issued_at']))
self.assertTrue(re.search('^[\\dTZ.:-]+$', body['issued_at']))
self.assertEqual('custId', body['uuid_type'])
# Same format as with no parameter
self.assertTrue(re.search('^[0-9a-f-]+$', body['uuid']))

6
orm/tests/unit/cms/test_customer.py
View File

@ -444,9 +444,9 @@ RET_CUSTOMER_JSON = {
members='1', pools='1', nat_instance='1', route_table='1', vips='1',
loadbalancer='1', listener='1')]
})]})],
"users": [Models.User(**
{"id": "userId1", "role": ["admin", "other"]})
],
"users": [Models.User(
**{"id": "userId1", "role": ["admin", "other"]})
],
"defaultQuotas": [Models.Quota(**{
"compute": [Models.Compute(instances='1', key_pairs='1', ram='1',
vcpus='1', metadata_items='1',

6
orm/tests/unit/fms/test_flavor_logic.py
View File

@ -2,7 +2,9 @@ from orm.common.orm_common.injector import injector
from orm.common.orm_common.utils import utils
from orm.services.flavor_manager.fms_rest.data.sql_alchemy import db_models
from orm.services.flavor_manager.fms_rest.data.wsme import models
from orm.services.flavor_manager.fms_rest.data.wsme.models import *
from orm.services.flavor_manager.fms_rest.data.wsme.models import (
ExtraSpecsWrapper, Flavor, FlavorWrapper,
Region, RegionWrapper, TagsWrapper, TenantWrapper)
from orm.services.flavor_manager.fms_rest.logic.error_base import NotFoundError
from orm.services.flavor_manager.fms_rest.logic import flavor_logic
from orm.tests.unit.fms import FunctionalTest
@ -156,7 +158,7 @@ class TestFlavorLogic(FunctionalTest):
get_extra_spec_needed = MagicMock()
get_extra_spec_needed.get_extra_spec_needed.return_value = [extra_spec_needed]
mock_flavorwrapper.from_db_model.return_value = get_extra_spec_needed
with self.assertRaises(ErrorStatus):
with self.assertRaises(flavor_logic.ErrorStatus):
flavor_logic.delete_extra_specs(123, "transaction_id", "key1")
@patch.object(flavor_logic, 'FlavorWrapper', return_value=MagicMock())

1
orm/tests/unit/rds/controllers/v1/resources/test_create_resource.py
View File

@ -124,6 +124,7 @@ class TestCreateResource(FunctionalTest):
headers=headers, expect_errors=True)
assert response.status_int == 409
good_data = {
"service_template": {
"resource": {

15
orm/tests/unit/rms/controllers/v1/orm/resources/test_metadata.py
View File

@ -3,17 +3,18 @@ import json
from unittest.mock import patch, MagicMock
from orm.services.region_manager.rms.controllers.v2.orm.resources import metadata
from orm.services.region_manager.rms.model.model import *
from orm.services.region_manager.rms.model import model
from orm.tests.unit.rms import FunctionalTest
from wsme.exc import ClientSideError
result_inst = RegionData("1", "2", "3", "4", "5", "6",
endpoints=[
EndPoint("http://www.example.co.il", "url")
],
address=Address("US", "NY", "HANEGEV", "AIRPORT_CITY", "5"),
metadata={"key1": ["value1"], "key2": ["value2"]})
result_inst = model.RegionData(
"1", "2", "3", "4", "5", "6",
endpoints=[
model.EndPoint("http://www.example.co.il", "url")
],
address=model.Address("US", "NY", "HANEGEV", "AIRPORT_CITY", "5"),
metadata={"key1": ["value1"], "key2": ["value2"]})
result_dict = {'status': '2', 'vlcpName': None, 'clli': '5',
'name': '4', 'designType': None,

1
orm/tests/unit/rms/db_testing.py
View File

@ -86,5 +86,6 @@ def main():
run_db_tests(data_manager)
if __name__ == "__main__":
main()

3
test-requirements.txt
View File

@ -2,7 +2,8 @@
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
hacking>=3.0.1,<3.1.0 # Apache-2.0
pep257==0.7.0 # MIT License
bandit==1.6.2 # Apache-2.0
coverage!=4.4,>=4.0 # Apache-2.0
openstackdocstheme>=1.18.1

43
tools/zuul/playbooks/docker-image-build.yaml
View File

@ -4,46 +4,9 @@
tasks:
- include_vars: vars.yaml
- name: Install Docker (Debian)
when: ansible_os_family == 'Debian'
block:
- file:
path: "{{ item }}"
state: directory
with_items:
- /etc/docker/
- /etc/systemd/system/docker.service.d/
- /var/lib/docker/
- mount:
path: /var/lib/docker/
src: tmpfs
fstype: tmpfs
opts: size=25g
state: mounted
- copy: "{{ item }}"
with_items:
- content: "{{ docker_daemon | to_json }}"
dest: /etc/docker/daemon.json
- src: files/docker-systemd.conf
dest: /etc/systemd/system/docker.service.d/
- apt_key:
url: https://download.docker.com/linux/ubuntu/gpg
- apt_repository:
repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker bionic stable
- apt:
name: "{{ item }}"
allow_unauthenticated: True
with_items:
- docker-ce
- pip:
name: docker
version: 2.7.0
- iptables:
action: insert
chain: INPUT
in_interface: docker0
jump: ACCEPT
become: True
- name: Ensure docker
include_role:
name: ensure-docker
- name: Make images
when: not publish

2
tox.ini
View File

@ -66,6 +66,6 @@ commands = oslo-config-generator --config-file=tools/config/ranger-config-genera
[flake8]
show-source = True
ignore = H301,F821,H202,H101,H104,H238,H401,H405,E501,F811,F403,H233,F841,H903
ignore = H301,F821,H202,H101,H104,H238,H401,H405,E501,F811,F403,H233,F841,H903,W503
builtins = _
exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build