Update RDS to create resources as specified in Ranger Region

Ranger region now includes a field which declares
which openstack domain a resource should be deployed
in. This change updates rds to create resources in
that domain via the heat template

Change-Id: I43144eb75a34661fae15399b9d32842d65327621

etc/policy.json

@@ -1,7 +1,7 @@
 {
     "default": "!",
 
-    "admin": "role:admin and tenant:admin or role:admin and tenant:service or role:admin and tenant:ranger",
+    "admin": "role:admin and tenant:admin or role:admin and tenant:service",
     "admin_support": "role:admin_support and tenant:admin or role:admin_support and tenant:service or role:admin_support and tenant:ranger",
     "admin_viewer": "role:admin_viewer and tenant:admin or role:admin_viewer and tenant:service or role:admin_viewer and tenant:ranger",
     "flavor_creator": "role:ranger_flavor_creator and tenant:admin or role:ranger_flavor_creator and tenant:service or role:ranger_flavor_creator and tenant:ranger",

orm/orm_client/ormcli/cmscli.py

@@ -608,7 +608,8 @@ def cmd_details(args):
 
 
 def validate_args(args):
-    for argument in ('tenant_name', 'username', 'password', 'auth_region', 'keystone_auth_url'):
+    for argument in ('tenant_name', 'username', 'password',
+                     'auth_region', 'keystone_auth_url'):
         argument_value = getattr(args, argument, None)
         if argument_value is not None:
             globals()[argument] = argument_value

orm/orm_client/ormcli/fmscli.py

@@ -296,7 +296,8 @@ def cmd_details(args):
 
 
 def validate_args(args):
-    for argument in ('tenant_name', 'username', 'password', 'auth_region', 'keystone_auth_url'):
+    for argument in ('tenant_name', 'username', 'password',
+                     'auth_region', 'keystone_auth_url'):
         argument_value = getattr(args, argument, None)
         if argument_value is not None:
             globals()[argument] = argument_value

orm/orm_client/ormcli/imscli.py

@@ -188,7 +188,8 @@ def add_to_parser(service_sub):
 
 
 def validate_args(args):
-    for argument in ('tenant_name', 'username', 'password', 'auth_region', 'keystone_auth_url'):
+    for argument in ('tenant_name', 'username', 'password',
+                     'auth_region', 'keystone_auth_url'):
         argument_value = getattr(args, argument, None)
         if argument_value is not None:
             globals()[argument] = argument_value

orm/orm_client/ormcli/rmscli.py

@@ -213,7 +213,8 @@ def add_to_parser(service_sub):
 
 
 def validate_args(args):
-    for argument in ('tenant_name', 'username', 'password', 'auth_region', 'keystone_auth_url'):
+    for argument in ('tenant_name', 'username', 'password',
+                     'auth_region', 'domain_name', 'keystone_auth_url'):
         argument_value = getattr(args, argument, None)
         if argument_value is not None:
             globals()[argument] = argument_value

orm/services/customer_manager/config.py

@@ -16,6 +16,11 @@ server = {
 
 # Pecan Application Configurations
 
+
+rds = {
+    'customer_domain': config.rds['customer_domain']
+}
+
 app = {
     'root': 'orm.services.customer_manager.cms_rest.controllers.root.RootController',
     'modules': ['orm.services.customer_manager.cms_rest'],

orm/services/db_setup.py

@@ -43,9 +43,11 @@ def execute_app_custom_sql(conn):
 
         sql = 'insert ignore into cms_domain(name) values(%s)'
         conn.execute(sql, (customer_domain, ))
-        update_regions = "update region set domain_name = '%s' " \
-            "where domain_name is NULL or length(domain_name) = 0"
-        conn.execute(update_regions, (customer_domain, ))
+        # TODO(jh629g): resolve errors with automated execution of
+        # sql command below
+        # update_regions = "update region set domain_name = '%s'" \
+        # " where domain_name is NULL or length(domain_name) = 0"
+        # conn.execute(update_regions, customer_domain)
 
 
 def execute_purge_uuids_record(conn):

orm/services/image_manager/config.py

@@ -14,6 +14,10 @@ server = {
 }
 
 # Pecan Application Configurations
+rds = {
+    'image_domain': config.rds['customer_domain']
+}
+
 app = {
     'root': 'orm.services.image_manager.ims.controllers.root.RootController',
     'modules': ['orm.services.image_manager.ims'],

orm/services/resource_distributor/config.py

@@ -48,6 +48,10 @@ rms = {
     'all_regions_path': 'v2/orm/regions'
 }
 
+rds = {
+    'project_domain': config.rds['customer_domain']
+}
+
 ordupdate = {
     'discovery_url': 'http://' + config.orm_host,
     'discovery_port': config.rms['port'],
@@ -151,6 +155,7 @@ region_resource_id_status = {
     }
 }
 
+
 app_module = app['modules'][0]
 logging = config.get_log_config(config.rds['log'],
                                 app['service_name'],

orm/services/resource_distributor/rds/ordupdate/ord_notifier.py

@@ -293,7 +293,6 @@ def notify_ord(transaction_id,
             message = 'Ranger Agent URL of LCP %s not found' % (region_id, )
             logger.error(message)
             raise OrdNotFoundError(Exception(message))
-
         if operation != 'create':
             record = regionResourceIdStatus.get_template_data(
                 resource_id, region_id)

orm/services/resource_distributor/rds/services/resource.py

@@ -117,6 +117,7 @@ def _create_template_data(input_data):
     jsondata = input_data.model
     targetslist = []
     targets = input_data.targets
+
     for target in targets:
         # save start status to submitted for each region
         _create_or_update_resource_status(input_data, target)
@@ -162,6 +163,7 @@ def _create_template_data(input_data):
         targetslist.append({"region_id": target['name'],
                             "resource_type": input_data.resource_type,
                             "resource_name": input_data.resource_id,
+                            "resource_domain": target['domain_name'],
                             "template_data": yamldata,
                             "operation": target['action']})
     return targetslist

orm/services/resource_distributor/rds/services/yaml_customer_builder.py

@@ -39,7 +39,9 @@ def yamlbuilder(alldata, region):
     resources = {}
     yaml_version = conf.yaml_configs.customer_yaml.yaml_version
     yaml_type = conf.yaml_configs.customer_yaml.yaml_options.type
-    domain = conf.yaml_configs.customer_yaml.customer_domain
+
+    auth_domain = conf.authentication.user_domain_name
+    domain = region['domain_name']
     title = {'heat_template_version': yaml_version}
     description = {'description': 'yaml file for region - %s' % region['name']}
     jsondata = alldata
@@ -106,7 +108,7 @@ def yamlbuilder(alldata, region):
         else:
             resources['resources'][user['id']] = \
                 {'type': 'OS::Keystone::UserRoleAssignment\n',
-                 'properties': {'user': (user['id'] + "{%s}" % domain),
+                 'properties': {'user': (user['id'] + "{%s}" % auth_domain),
                                 'roles': user_roles}}
 
         # create the output for users

orm/services/resource_distributor/rds/services/yaml_flavor_builder.py

@@ -76,5 +76,5 @@ def yamlbuilder(alldata, region):
         {'value': {"get_resource": flavor_type}}
     flavor_yaml = create_final_yaml(title, resources, description, outputs)
     my_logger.debug(
-        "done!!! building flavor yaml for region %s " % region['name'])
+        "Done building flavor yaml for region %s" % region['name'])
     return flavor_yaml

orm/services/resource_distributor/rds/utils/authentication.py

@@ -3,6 +3,7 @@ import logging
 import requests
 
 from orm.common.client.keystone.keystone_utils import tokens
+from orm.services.resource_distributor import config
 from orm.services.resource_distributor.rds.storage import factory
 
 from pecan import conf
@@ -13,8 +14,7 @@ mech_password = False
 rms_url = ""
 tenant_name = ""
 keystone_version = ""
-user_domain_name = "default"
-project_domain_name = "default"
+project_domain_name = config.rds['project_domain']
 
 
 headers = {'content-type': 'application/json'}
@@ -29,7 +29,7 @@ def _is_authorization_enabled():
 
 def _get_token_conf():
     conf = tokens.TokenConf(mech_id, mech_password, rms_url, tenant_name,
-                            keystone_version, user_domain_name, project_domain_name)
+                            keystone_version, config.authentication.user_domain_name, project_domain_name)
     return conf
 
 
@@ -75,7 +75,7 @@ def get_token(region):
                     "user": {
                         "name": conf.authentication.mech_id,
                         "domain": {
-                            "name": user_domain_name
+                            "name": conf.authentication.user_domain_name
                         },
                         "password": conf.authentication.mech_pass
                     }
@@ -84,7 +84,7 @@ def get_token(region):
             "scope": {
                 "project": {
                     "domain": {
-                        "name": project_domain_name
+                        "name": conf.authentication.user_domain_name
                     },
                     "name": conf.authentication.tenant_name
                 }

orm/services/resource_distributor/rds/utils/utils.py

@@ -87,7 +87,8 @@ def add_rms_status_to_regions(resource_regions, resource_type):
     # iterate through rms regions and get regions status and version
     for region in all_regions['regions']:
         rms_regions[region['name']] = {'status': region['status'],
-                                       'version': region['rangerAgentVersion']}
+                                       'version': region['rangerAgentVersion'],
+                                       'domain_name': region['domainName']}
 
     # iterate through resource regions and add to them rms status
     for region in resource_regions:
@@ -101,6 +102,7 @@ def add_rms_status_to_regions(resource_regions, resource_type):
                         region['name'], supported_resource_version[0] if supported_resource_version else '0'))
 
             region['rms_status'] = rms_regions[region['name']]['status']
+            region['domain_name'] = rms_regions[region['name']]['domain_name']
             continue
         # if region not found in rms
         region['rms_status'] = "region_not_found_in_rms"

orm/tests/unit/ormcli/test_cmscli.py

@@ -172,9 +172,6 @@ class CmsTests(TestCase):
         with self.assertRaises(SystemExit) as cm:
             args = ormcli.main('orm cms list_customers t'.split())
         self.assertEqual(cm.exception.code, 1)
-        sys.stdout.seek(0)
-        output = sys.stdout.read()
-        self.assertIn('e', output)
 
     @mock.patch.object(cmscli, 'validate_args')
     @mock.patch.object(cmscli.requests, 'post')

orm/tests/unit/rds/services/test_create_resource.py

@@ -46,30 +46,6 @@ class InputData(object):
 class CreateResource(unittest.TestCase):
     """create resource test."""
 
-    # @patch.object(ResourceService.regionResourceIdStatus,
-    #               'get_regions_by_status_resource_id',
-    #               return_value=StatusModel(status=[result]))
-    # def test_create_customer_conflict_rise(self, result):
-    #     """check raise conflict."""
-    #     with self.assertRaises(ResourceService.ConflictValue):
-    #         ResourceService.main(jsondata, uuid, 'customer', 'create')
-
-    # @patch.object(ResourceService, '_upload_to_sot', return_value=[1, 2])
-    # @patch.object(ResourceService, '_create_data_to_sot', return_value=[1, 2])
-    # @patch.object(ResourceService.regionResourceIdStatus,
-    #               'get_regions_by_status_resource_id', return_value=None)
-    # @patch.object(ResourceService.uuid_utils,
-    #               'get_random_uuid', return_value='uuid-gen-123456')
-    # def test_create_customer_valid_uuid_gen(self, tranid, result,
-    #                                         sotdata, sotupload):
-    #     """check flow with uuid gen."""
-    #     status_model = StatusModel(status=[result])
-    #     status_model.regions = None
-    #     result.return_value = status_model
-    #     resource_id = ResourceService.main(jsondata, uuid,
-    #                                        'customer', 'create')
-    #     self.assertEqual(resource_id, jsondata['uuid'])
-
     @patch.object(ResourceService.regionResourceIdStatus, 'add_status',
                   return_value=None)
     @patch.object(ResourceService, '_submit_template_data', return_value=[1, 2])
@@ -88,61 +64,6 @@ class CreateResource(unittest.TestCase):
             resource_id = ResourceService.main(jsondata, uuid,
                                                'customer', 'create')
 
-    # @patch.object(ResourceService.regionResourceIdStatus, 'add_status',
-    #               return_value=None)
-    # @patch.object(ResourceService.yaml_customer_builder, 'yamlbuilder',
-    #               return_value=["anystring"])
-    # @patch.object(ResourceService, '_upload_to_sot', return_value=[1, 2])
-    # @patch.object(ResourceService.regionResourceIdStatus,
-    #               'get_regions_by_status_resource_id', return_value=None)
-    # @patch.object(ResourceService.uuid_utils, 'get_random_uuid',
-    #               return_value='uuid-gen-123456')
-    # def test_create_customer_sot_data(self, tranid, result, sotupload,
-    #                                   yamlbuilder, database):
-    #     """check sot data build for customer."""
-    #     status_model = StatusModel(status=[result])
-    #     status_model.regions = None
-    #     result.return_value = status_model
-    #     resource_id = ResourceService.main(jsondata, uuid,
-    #                                        'customer', 'create')
-
-    # @patch.object(ResourceService.regionResourceIdStatus, 'add_status',
-    #               return_value=None)
-    # @patch.object(ResourceService.yaml_customer_builder, 'yamlbuilder',
-    #               return_value=["anystring"])
-    # @patch.object(ResourceService.sot_factory, 'get_sot',
-    #               return_value=SoT())
-    # @patch.object(ResourceService.regionResourceIdStatus,
-    #               'get_regions_by_status_resource_id', return_value=None)
-    # @patch.object(ResourceService.uuid_utils, 'get_random_uuid',
-    #               return_value='uuid-gen-123456')
-    # def test_create_resource_upload_sot(self, tranid, result, sotupload,
-    #                                     yamlbuilder, database):
-    #     """check upload to sot."""
-    #     status_model = StatusModel(status=[result])
-    #     status_model.regions = None
-    #     result.return_value = status_model
-    #     resource_id = ResourceService.main(jsondata, uuid,
-    #                                        'customer', 'create')
-
-    # @patch.object(ResourceService.regionResourceIdStatus, 'add_status',
-    #               return_value=None)
-    # @patch.object(ResourceService.yaml_flavor_builder, 'yamlbuilder',
-    #               return_value=["anystring"])
-    # @patch.object(ResourceService.sot_factory, 'get_sot', return_value=SoT())
-    # @patch.object(ResourceService.regionResourceIdStatus,
-    #               'get_regions_by_status_resource_id', return_value=None)
-    # @patch.object(ResourceService.uuid_utils,
-    #               'get_random_uuid', return_value='uuid-gen-123456')
-    # def test_create_flavor_sot_data(self, tranid, result, sotupload,
-    #                                 yamlbuilder, database):
-    #     """check flavor data create."""
-    #     status_model = StatusModel(status=[result])
-    #     status_model.regions = None
-    #     result.return_value = status_model
-    #     resource_id = ResourceService.main(flavorjsondata, uuid,
-    #                                        'flavor', 'create')
-
     @patch.object(ResourceService.regionResourceIdStatus,
                   'add_status', return_value=None)
     @patch.object(ResourceService.yaml_customer_builder,
@@ -151,7 +72,8 @@ class CreateResource(unittest.TestCase):
                   'get_regions_by_status_resource_id', return_value=None)
     @patch.object(ResourceService.uuid_utils, 'get_random_uuid',
                   return_value='uuid-gen-123456')
-    def test_create_flavor_data_check(self, tranid, result,
+    def test_create_flavor_data_check(self,
+                                      tranid, result,
                                       yamlbuilder, database):
         """check list creating."""
         input_data = InputData(
@@ -193,68 +115,6 @@ class CreateResource(unittest.TestCase):
             resource_id = ResourceService.main(flavorjsondata, uuid,
                                                'flavor', 'delete')
 
-    # @patch.object(ResourceService.yaml_flavor_builder,
-    #               'yamlbuilder', return_value=["anystring"])
-    # @patch.object(ResourceService.regionResourceIdStatus,
-    #               'add_status', return_value=None)
-    # # @patch.object(ResourceService, '_delete_from_sot', return_value = None)
-    # @patch.object(ResourceService.sot_factory, 'get_sot', return_value=SoT())
-    # @patch.object(ResourceService.regionResourceIdStatus,
-    #               'get_regions_by_status_resource_id', return_value=None)
-    # @patch.object(ResourceService.uuid_utils, 'get_random_uuid',
-    #               return_value='uuid-gen-123456')
-    # def test_delete_flavor_not_valid_all(self, tranid, result,
-    #                                      sotdata, sotupload, yaml_mock):
-    #     """delete flavor uuid gen raise an error."""
-    #     status_model = StatusModel(status=[result])
-    #     status_model.regions = None
-    #     result.return_value = status_model
-    #     resource_id = ResourceService.main(flavorjsondata, uuid,
-    #                                        'flavor', 'delete')
-    #     self.assertEqual('uuid-uuid-uuid-uuid', resource_id)
-
-    # @patch.object(ResourceService.regionResourceIdStatus, 'add_status',
-    #               return_value=None)
-    # @patch.object(ResourceService.yaml_customer_builder, 'yamlbuilder',
-    #               return_value=["anystring"])
-    # @patch.object(ResourceService.sot_factory, 'get_sot',
-    #               return_value=SoT())
-    # @patch.object(ResourceService.regionResourceIdStatus,
-    #               'get_regions_by_status_resource_id', return_value=None)
-    # @patch.object(ResourceService.uuid_utils, 'get_random_uuid',
-    #               return_value='uuid-gen-123456')
-    # def test_create_resource_up2load_sot_put(self, moc_get_random_uuid,
-    #                                          moc_get_regions_by_status_resource_id,
-    #                                          moc_get_sot,
-    #                                          moc_yamlbuilder, moc_add_status):
-    #     """check upload to sot."""
-    #     status_model = StatusModel(status=[result])
-    #     status_model.regions = None
-    #     moc_get_regions_by_status_resource_id.return_value = status_model
-    #     resource_id = ResourceService.main(jsondata, uuid,
-    #                                        'customer', 'modify')
-
-    # @patch.object(ResourceService.regionResourceIdStatus, 'add_status',
-    #               return_value=None)
-    # @patch.object(ResourceService.yaml_image_builder, 'yamlbuilder',
-    #               return_value=["anystring"])
-    # @patch.object(ResourceService.sot_factory, 'get_sot',
-    #               return_value=SoT())
-    # @patch.object(ResourceService.regionResourceIdStatus,
-    #               'get_regions_by_status_resource_id', return_value=None)
-    # @patch.object(ResourceService.uuid_utils, 'get_random_uuid',
-    #               return_value='uuid-gen-123456')
-    # def test_create_resource_up2load_sot_put_image(self, moc_get_random_uuid,
-    #                                          moc_get_regions_by_status_resource_id,
-    #                                          moc_get_sot,
-    #                                          moc_yamlbuilder, moc_add_status):
-    #     """check upload to sot."""
-    #     status_model = StatusModel(status=[result])
-    #     status_model.regions = None
-    #     moc_get_regions_by_status_resource_id.return_value = status_model
-    #     resource_id = ResourceService.main(json_data_image, uuid,
-    #                                        'image', 'modify')
-
     def test_get_inputs_from_resource_type(self):
         input_data = ResourceService._get_inputs_from_resource_type(jsondata,
                                                                     'customer',
@@ -510,20 +370,27 @@ json_data = {
     "name": "welcome_man"
 }
 
-target_list = [{'template_data': ['anystring'],
-                'operation': 'create',
+target_list = [{'region_id': 'regionname',
+                'resource_type': 'customer',
                 'resource_name': '1e24981a-fa51-11e5-86aa-5e5517507c66',
-                'region_id': 'regionname', 'resource_type': 'customer'},
-               {'template_data': 'delete', 'operation': 'delete',
+                'resource_domain': 'default',
+                'template_data': ['anystring'],
+                'operation': 'create'},
+               {'region_id': 'regionnametest',
                 'resource_name': '1e24981a-fa51-11e5-86aa-5e5517507c66',
-                'region_id': 'regionnametest', 'resource_type': 'customer'}]
+                'resource_type': 'customer',
+                'resource_domain': 'default',
+                'template_data': 'delete',
+                'operation': 'delete'}]
 
-targets = [{'users': [{'id': 'userId1', 'roles': ['admin', 'other']},
+targets = [{'name': 'regionname', "action": "create",
+            'users': [{'id': 'userId1', 'roles': ['admin', 'other']},
                       {'id': 'userId2', 'roles': ['storage']}],
-            'name': 'regionname', "action": "create", 'quotas': []},
-           {'users': [],
-            'name': 'regionnametest',
-            "action": "delete",
+            'domain_name': 'default',
+            'quotas': []},
+           {'name': 'regionnametest', "action": "delete",
+            'users': [],
+            'domain_name': 'default',
             'quotas': [{'storage': {'gigabytes': '10', 'volumes': '10',
                                     'snapshots': '10'},
                         'compute': {'instances': '10', 'ram': '10',

orm/tests/unit/rds/services/test_customer_yaml.py

@@ -27,7 +27,8 @@ alldata = {
     'description': 'this is a description', 'enabled': 1,
     'regions': [{'users': [{'id': 'userId1', 'roles': ['admin', 'other']},
                            {'id': 'userId2', 'roles': ['storage']}],
-                 'name': 'regionname', 'quotas': []},
+                 'name': 'regionname',
+                 'quotas': []},
                 {'users': [], 'name': 'regionnametest',
                  'quotas': [{'storage': {'gigabytes': '10',
                                          'volumes': '10',
@@ -43,6 +44,7 @@ region_quotas = {'users':
                  [],
                  'name': 'regionnametest',
                  'rangerAgentVersion': 3.0,
+                 'domain_name': 'default',
                  'quotas': [{'storage': {'gigabytes': '10',
                                          'volumes': '10', 'snapshots': '10'},
                              'compute': {'instances': '10', 'ram': '10',
@@ -55,16 +57,19 @@ region_quotas = {'users':
 
 region_users = {'users': [{'id': 'userId1', 'roles': ['admin', 'other']},
                           {'id': 'userId2', 'roles': ['storage']}],
+                'domain_name': 'default',
                 'name': 'regionname', 'rangerAgentVersion': 3.0, 'quotas': []}
 
 region_users_v4 = {'users': [{'id': 'userId1', 'roles': ['admin', 'other']},
                              {'id': 'userId2', 'roles': ['storage']}],
+                   'domain_name': 'default',
                    'name': 'regionname', 'rangerAgentVersion': 4.0, 'quotas': []}
 
 full_region = {'users': [{'id': 'userId1', 'roles': ['admin', 'other']},
                          {'id': 'userId2', 'roles': ['storage']}],
                'name': 'regionnametest',
                'rangerAgentVersion': 3.0,
+               'domain_name': 'default',
                'quotas': [{'storage': {'gigabytes': '10',
                                        'volumes': '10', 'snapshots': '10'},
                            'compute': {'instances': '10', 'ram': '10',
@@ -325,6 +330,8 @@ class CreateResource(unittest.TestCase):
         """test valid dict to yaml output as expected with ldap system."""
         ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-2'
         domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
+        mock_conf.yaml_configs.rds.project_domain = 'default'
+        mock_conf.authentication.user_domain_name = 'default'
         mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = False
         mock_conf.yaml_configs.customer_yaml.yaml_options.type = "ldap"
         yamlfile = CustomerBuild.yamlbuilder(alldata, region_users)