Add customer's domain attribute in CMS API call

When CMS API call is made for either create and update customer, a

A new optional customer domain field is introduced for CMS API call
to create or update customer. The customer domain value will be used
by Rds to set the customer and users domain value when generating the
heat template.  In the event customer domain is not provided by the
CMS API call, a default value will be retrieved from the config file.

Change-Id: I58dbae760d56b756b72c5d9eaf7c0b61cf193847

orm/services/customer_manager/cms_rest/data/data_manager.py

@@ -52,7 +52,8 @@ class DataManager(object):
         if not connection_string:
             connection_string = conf.database.connection_string
 
-        self._engine_facade = LegacyEngineFacade(connection_string, autocommit=False)
+        self._engine_facade = LegacyEngineFacade(connection_string,
+                                                 autocommit=False)
         self._session = None
         listen(self.session, 'before_flush', on_before_flush)
         self.image_record = None
@@ -245,7 +246,8 @@ class DataManager(object):
             uuid=uuid,
             name=customer.name,
             enabled=customer.enabled,
-            description=customer.description
+            description=customer.description,
+            customer_domain=customer.customerDomain
         )
 
         self.session.add(sql_customer)

orm/services/customer_manager/cms_rest/data/sql_alchemy/models.py

@@ -491,6 +491,7 @@ class Customer(Base, CMSBaseModel):
     uuid = Column(String(64), nullable=False, unique=True)
     name = Column(String(64), nullable=False, unique=True)
     description = Column(String(255), nullable=False)
+    customer_domain = Column(String(64), nullable=False)
     enabled = Column(SmallInteger, nullable=False)
     customer_customer_regions = relationship("CustomerRegion", cascade="all, delete, delete-orphan")
     customer_metadata = relationship("CustomerMetadata", cascade="all, delete, delete-orphan")
@@ -501,6 +502,7 @@ class Customer(Base, CMSBaseModel):
             uuid=self.uuid,
             name=self.name,
             description=self.description,
+            customer_domain=self.customer_domain,
             enabled=self.enabled,
             customer_customer_regions=[customer_region.__json__() for customer_region in
                                        self.customer_customer_regions],
@@ -515,6 +517,7 @@ class Customer(Base, CMSBaseModel):
             "uuid": self.uuid,
             "name": self.name,
             "description": self.description,
+            "customer_domain": self.customer_domain,
             "enabled": 1 if self.enabled else 0
         }
 
@@ -558,6 +561,7 @@ class Customer(Base, CMSBaseModel):
                                      name=name,
                                      regions=regions,
                                      users=defaultRegion[0].users if defaultRegion else [],
+                                     customerDomain=self.customer_domain,
                                      metadata=metadata,
                                      defaultQuotas=defaultRegion[0].quotas if defaultRegion else [],
                                      custId=self.uuid,

orm/services/customer_manager/cms_rest/logic/customer_logic.py

@@ -27,6 +27,9 @@ class CustomerLogic(object):
         if customer.name.strip() == '':
             raise ErrorStatus(400, "Customer Name can not be blank.")
 
+        if not customer.customerDomain:
+            customer.customerDomain = conf.rds.customer_domain
+
         try:
             sql_customer = datamanager.add_customer(customer, uuid)
 
@@ -165,7 +168,6 @@ class CustomerLogic(object):
             customer.handle_region_group()
             sql_customer = self.build_full_customer(customer, uuid, datamanager)
             customer_result_wrapper = build_response(uuid, transaction_id, 'create')
-
             if sql_customer.customer_customer_regions and len(sql_customer.customer_customer_regions) > 1:
                 customer_dict = sql_customer.get_proxy_dict()
                 for region in customer_dict["regions"]:

orm/services/customer_manager/cms_rest/model/Models.py

@@ -183,7 +183,7 @@ class User(Model):
         """Create a new compute.
 
         :param id:  user id
-        :param role:  roles this use belong to
+        :param role:  roles this user belong to
         """
         self.id = id
         self.role = role
@@ -229,13 +229,14 @@ class Customer(Model):
     metadata = wsme.wsattr(wsme.types.DictType(str, str), mandatory=False)
     regions = wsme.wsattr([Region], mandatory=False)
     users = wsme.wsattr([User], mandatory=True)
+    customerDomain = wsme.wsattr(str, mandatory=False)
     defaultQuotas = wsme.wsattr([Quota], mandatory=True)
     status = wsme.wsattr(wsme.types.text, mandatory=False)
     custId = wsme.wsattr(wsme.types.text, mandatory=False)
     uuid = wsme.wsattr(wsme.types.text, mandatory=False)
 
     def __init__(self, description="", enabled=False, name="", metadata={}, regions=[], users=[],
-                 defaultQuotas=[], status="", custId="", uuid=None):
+                 customerDomain="", defaultQuotas=[], status="", custId="", uuid=None):
         """Create a new Customer.
 
         :param description:  Server name
@@ -248,6 +249,7 @@ class Customer(Model):
         self.metadata = metadata
         self.regions = regions
         self.users = users
+        self.customerDomain = customerDomain
         self.defaultQuotas = defaultQuotas
         self.status = status
         self.custId = custId

orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql

@@ -30,6 +30,7 @@ create table if not exists customer
 	 uuid varchar(64) not null,
  	 name varchar(64) not null,
 	 description varchar(255) not null,
+	 customer_domain varchar(64) not null,
 	 enabled tinyint not null,
 	 primary key (id),
 	 unique uuid_idx (uuid),

orm/services/customer_manager/scripts/db_scripts/ranger_cms_update_db.sql

@@ -6,7 +6,6 @@ insert ignore into cms_region(id,name,type) values(-1, "DEFAULT", "single");
 insert ignore into cms_domain(id,name) values(1, "default");
 
 DROP PROCEDURE IF EXISTS MoveKeyToQuota;
-DELIMITER ;;
 CREATE PROCEDURE `MoveKeyToQuota`(p_field_key varchar(64), p_quota varchar(64))
 BEGIN
   DECLARE bDone INT;
@@ -72,8 +71,7 @@ BEGIN
   CLOSE security_elements;
 
   COMMIT;
-END;;
+END;
-DELIMITER ;
 
 CALL MoveKeyToQuota('security_groups', 'network');
 CALL MoveKeyToQuota('security_group_rules', 'network');
@@ -82,11 +80,9 @@ SELECT "==========================" as "";
 SELECT q.*, qfd.* FROM quota_field_detail qfd
 	left join quota q on (q.id = qfd.quota_id) where qfd.field_key like "security%%";
 
-DELIMITER ;;
 
 # account for the old procedure name in DROP PROCEDURE before the name change
-DROP PROCEDURE IF EXISTS add_regoion_type ;;
+DROP PROCEDURE IF EXISTS add_region_type;
-DROP PROCEDURE IF EXISTS add_region_type ;;
 CREATE PROCEDURE add_region_type()
 BEGIN
 
@@ -96,7 +92,18 @@ BEGIN
   	insert ignore into cms_region(id,name,type) values(-1, "DEFAULT", "single");
 	END IF;
 
-END ;;
+END;
-DELIMITER ;
 
 CALL add_region_type();
+
+
+DROP PROCEDURE IF EXISTS add_customer_domain;
+CREATE PROCEDURE add_customer_domain()
+BEGIN
+   IF NOT EXISTS( (SELECT * FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE()
+      AND COLUMN_NAME='customer_domain' AND TABLE_NAME='customer') ) THEN
+      ALTER TABLE customer ADD customer_domain varchar(64);
+   END IF;
+END;
+
+CALL add_customer_domain();

orm/services/db_setup.py

@@ -36,17 +36,24 @@ def execute_app_custom_sql(conn):
     config = ConfigParser()
     config.read(CONF.config_file)
 
-    # Insert custom domain name into cms_domain.
     if config.has_option("rds", "customer_domain"):
         customer_domain = config.get("rds", "customer_domain")
         customer_domain = re.sub(r'[\'\"]', '', customer_domain).strip()
 
+        # Insert custom domain name into cms_domain talbe.
         sql = 'insert ignore into cms_domain(name) values(%s)'
         conn.execute(sql, (customer_domain, ))
+
+        # Update domain_name column in region table
         update_regions = "update region set domain_name = %s" \
             " where domain_name is NULL or length(domain_name) = 0"
         conn.execute(update_regions, customer_domain)
 
+        # Update users_domain column in customer table
+        update_customer_domain = "update customer set customer_domain = %s" \
+            " where customer_domain is NULL or length(customer_domain) = 0"
+        conn.execute(update_customer_domain, customer_domain)
+
 
 def execute_purge_uuids_record(conn):
     """Execute custom SQL statements to purge records from uuids table.

orm/services/resource_distributor/rds/controllers/v1/resources/root.py

@@ -183,17 +183,12 @@ class CreateNewResource(rest.RestController):
         uuid = resource.service_template.tracking.tracking_id
         resource_type = resource.service_template.resource.resource_type
         base_url = pecan.request.application_url
-        user_data = {}
-        if resource_type == 'customer':
-            user_data['user_domain'] = pecan.request.headers['User-Domain']
-            user_data['user_id'] = pecan.request.headers['X-RANGER-Client']
         jsondata = ast.literal_eval(jsondata)
         try:
             resource_id = ResourceService.main(jsondata,
                                                uuid,
                                                resource_type,
-                                               'create',
+                                               'create')
-                                               user_data)
             site_link = "%s/v1/rds/%s/%s" % (base_url,
                                              resource_type,
                                              resource_id)
@@ -229,17 +224,12 @@ class CreateNewResource(rest.RestController):
         resource_type = resource.service_template.resource.resource_type
         base_url = pecan.request.application_url
         jsondata = ast.literal_eval(jsondata)
-        user_data = {}
-        if resource_type == 'customer':
-            user_data['user_domain'] = pecan.request.headers['User-Domain']
-            user_data['user_id'] = pecan.request.headers['X-RANGER-Client']
 
         try:
             resource_id = ResourceService.main(jsondata,
                                                uuid,
                                                resource_type,
-                                               'modify',
+                                               'modify')
-                                               user_data)
             my_logger.debug("data sent")
             site_link = "%s/v1/rds/%s/%s" % (base_url,
                                              resource_type,
@@ -275,10 +265,7 @@ class CreateNewResource(rest.RestController):
         jsondata = ast.literal_eval(jsondata)
         resource_uuid = resource.service_template.tracking.tracking_id
         resource_type = resource.service_template.resource.resource_type
-        user_data = {}
+
-        if resource_type == 'customer':
-            user_data['user_domain'] = pecan.request.headers['User-Domain']
-            user_data['user_id'] = pecan.request.headers['X-RANGER-Client']
         if resource_type not in resources_operation_list or operation not in \
                 resources_operation_list[resource_type]:
             raise NotAllowedError("delete Not allowed for this"
@@ -287,8 +274,7 @@ class CreateNewResource(rest.RestController):
             resource_id = ResourceService.main(jsondata,
                                                resource_uuid,
                                                resource_type,
-                                               operation,
+                                               operation)
-                                               user_data)
             return resource_id
         except ConflictValue as e:
             my_logger.error("the request blocked need to wait"

orm/services/resource_distributor/rds/services/resource.py

@@ -126,8 +126,7 @@ def _create_template_data(input_data):
         if target['action'] == "delete":
             yamldata = "delete"
         elif input_data.resource_type == "customer":
-            yamldata = yaml_customer_builder.yamlbuilder(jsondata, target,
+            yamldata = yaml_customer_builder.yamlbuilder(jsondata, target)
-                                                         input_data.user_data)
         elif input_data.resource_type == "group":
             yamldata = yaml_group_builder.yamlbuilder(jsondata, target)
         elif input_data.resource_type == "flavor":
@@ -164,7 +163,6 @@ def _create_template_data(input_data):
         targetslist.append({"region_id": target['name'],
                             "resource_type": input_data.resource_type,
                             "resource_name": input_data.resource_id,
-                            "resource_domain": target['domain_name'],
                             "template_data": yamldata,
                             "operation": target['action']})
     return targetslist
@@ -264,7 +262,7 @@ def _generate_resource_data(input_data):
 
 # User domain is only used in the case that a customer template is being generated,
 #  as certain builds of heat require user_domain in order to validate roles
-def main(jsondata, external_transaction_id, resource_type, operation, user_data=None):
+def main(jsondata, external_transaction_id, resource_type, operation):
     """main function handle resource operation."""
     my_logger.info("got %s for %s resource" % (operation, resource_type))
     try:
@@ -277,7 +275,6 @@ def main(jsondata, external_transaction_id, resource_type, operation, user_data=
         my_logger.debug("iterate through the regions see if none in submitted")
         _check_resource_status(input_data)
         my_logger.debug("get uuid from uuid generator")
-        input_data.user_data = user_data
         input_data.transaction_id = uuid_utils.get_random_uuid()
         my_logger.debug("uuid ={}".format(input_data.transaction_id))
         # add regions status from rms (to check if it down)

orm/services/resource_distributor/rds/services/yaml_customer_builder.py

@@ -26,9 +26,9 @@ def _metadata_to_tags(metadata):
         str(k) + '=' + str(v) for i in metadata for k, v in i.items()) + ']'
 
 
-def yamlbuilder(alldata, region, user_data):
+def yamlbuilder(alldata, region):
     logger.info("building customer yaml")
-    logger.debug("start building flavor yaml for region %s" % region['name'])
+    logger.debug("start building customer yaml for region %s" % region['name'])
     """build cstomer yaml.
 
     build yaml file from json
@@ -40,8 +40,7 @@ def yamlbuilder(alldata, region, user_data):
     resources = {}
     yaml_version = conf.yaml_configs.customer_yaml.yaml_version
     yaml_type = conf.yaml_configs.customer_yaml.yaml_options.type
-
+    domain = alldata['customer_domain']
-    domain = region['domain_name']
     title = {'heat_template_version': yaml_version}
     description = {'description': 'yaml file for region - %s' % region['name']}
     jsondata = alldata
@@ -105,16 +104,10 @@ def yamlbuilder(alldata, region, user_data):
                                 'groups': user_group,
                                 'roles': user_roles}}
         else:
-            if user['id'] == user_data['user_id']:
+            resources['resources'][user['id']] = \
-                resources['resources'][user['id']] = \
+                {'type': 'OS::Keystone::UserRoleAssignment\n',
-                    {'type': 'OS::Keystone::UserRoleAssignment\n',
+                 'properties': {'user': (user['id'] + "{%s}" % domain),
-                     'properties': {'user': (user['id'] + "{%s}" % user_data['user_domain']),
+                                'roles': user_roles}}
-                                    'roles': user_roles}}
-            else:
-                resources['resources'][user['id']] = \
-                    {'type': 'OS::Keystone::UserRoleAssignment\n',
-                     'properties': {'user': (user['id'] + "{%s}" % domain),
-                                    'roles': user_roles}}
 
         # create the output for users
         outputs['outputs']["%s_id" % user['id']] = \

orm/tests/unit/cms/test_customer_logic.py

@@ -85,6 +85,7 @@ class TestCustomerLogic(FunctionalTest):
     def test_create_customer_success_with_regions(self):
         customer.regions = [models.Region(name="a")]
         customer.name = 'Cust Name'
+        customer.customerDomain = 'domain'
         logic = customer_logic.CustomerLogic()
         logic.create_customer(customer, 'some_uuid', 'some_trans_id')
 
@@ -103,6 +104,7 @@ class TestCustomerLogic(FunctionalTest):
 
     def test_create_customer_add_all_default_users(self):
         customer.name = 'Cust Name'
+        customer.customerDomain = 'domain'
         logic = customer_logic.CustomerLogic()
         logic.create_customer(customer, 'some_uuid', 'some_trans_id')
 
@@ -113,6 +115,7 @@ class TestCustomerLogic(FunctionalTest):
         mock_returns_error = True
 
         customer.name = 'Cust Name'
+        customer.customerDomain = 'domain'
         logic = customer_logic.CustomerLogic()
 
         self.assertRaises(SystemError, logic.create_customer, customer, 'some_uuid', 'some_trans_id')
@@ -130,6 +133,7 @@ class TestCustomerLogic(FunctionalTest):
 
     def test_update_customer_success(self):
         customer.name = 'Cust Name'
+        customer.customerDomain = 'domain'
         logic = customer_logic.CustomerLogic()
         logic.update_customer(customer, 'some_uuid', 'some_trans_id')
 
@@ -189,6 +193,7 @@ class TestCustomerLogic(FunctionalTest):
         default_region = models.Region()
         customer.regions = [default_region]
         customer.name = 'Cust Name'
+        customer.customerDomain = 'domain'
 
         default_user = models.User()
         default_user.role = ['user', 'admin']

orm/tests/unit/rds/services/test_create_resource.py

@@ -374,13 +374,11 @@ json_data = {
 target_list = [{'region_id': 'regionname',
                 'resource_type': 'customer',
                 'resource_name': '1e24981a-fa51-11e5-86aa-5e5517507c66',
-                'resource_domain': 'default',
                 'template_data': ['anystring'],
                 'operation': 'create'},
                {'region_id': 'regionnametest',
                 'resource_name': '1e24981a-fa51-11e5-86aa-5e5517507c66',
                 'resource_type': 'customer',
-                'resource_domain': 'default',
                 'template_data': 'delete',
                 'operation': 'delete'}]
 

orm/tests/unit/rds/services/test_customer_yaml.py

@@ -25,6 +25,7 @@ alldata = {
                                                'network': '111',
                                                'floatingip': '111'}}]},
     'description': 'this is a description', 'enabled': 1,
+    'customer_domain': 'default',
     'regions': [{'users': [{'id': 'userId1', 'roles': ['admin', 'other']},
                            {'id': 'userId2', 'roles': ['storage']}],
                  'name': 'regionname',
@@ -287,8 +288,7 @@ class CreateResource(unittest.TestCase):
         ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-1'
         domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
         mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = False
-        user_data = {'user_domain': 'default', 'user_id': 'user1'}
+        yamlfile = CustomerBuild.yamlbuilder(alldata, region_quotas)
-        yamlfile = CustomerBuild.yamlbuilder(alldata, region_quotas, user_data)
         yamlfile_as_json = yaml.safe_load(yamlfile)
         self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
         self.assertEqual(yaml.safe_load(yamlfile), yaml.safe_load(fullyaml_no_users_quotasoff))
@@ -299,8 +299,7 @@ class CreateResource(unittest.TestCase):
         ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-2'
         domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
         mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = False
-        user_data = {'user_domain': 'default', 'user_id': 'user1'}
+        yamlfile = CustomerBuild.yamlbuilder(alldata, region_users)
-        yamlfile = CustomerBuild.yamlbuilder(alldata, region_users, user_data)
         yamlfile_as_json = yaml.safe_load(yamlfile)
         self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
         self.assertEqual(yaml.safe_load(yamlfile), yaml.safe_load(fullyaml_with_users_quotasoff))
@@ -311,8 +310,7 @@ class CreateResource(unittest.TestCase):
         ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-1'
         domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
         mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = True
-        user_data = {'user_domain': 'default', 'user_id': 'user1'}
+        yamlfile = CustomerBuild.yamlbuilder(alldata, region_users)
-        yamlfile = CustomerBuild.yamlbuilder(alldata, region_users, user_data)
         yamlfile_as_json = yaml.safe_load(yamlfile)
         self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
         self.assertEqual(yaml.safe_load(yamlfile), yaml.safe_load(full_yaml_default_quotas))
@@ -323,8 +321,7 @@ class CreateResource(unittest.TestCase):
         ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-1'
         domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
         mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = True
-        user_data = {'user_domain': 'default', 'user_id': 'user1'}
+        yamlfile = CustomerBuild.yamlbuilder(alldata, region_quotas)
-        yamlfile = CustomerBuild.yamlbuilder(alldata, region_quotas, user_data)
         yamlfile_as_json = yaml.safe_load(yamlfile)
         self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
         self.assertEqual(yaml.safe_load(yamlfile), yaml.safe_load(full_yaml_quotas))
@@ -338,8 +335,7 @@ class CreateResource(unittest.TestCase):
         mock_conf.authentication.user_domain_name = 'default'
         mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = False
         mock_conf.yaml_configs.customer_yaml.yaml_options.type = "ldap"
-        user_data = {'user_domain': 'default', 'user_id': 'user1'}
+        yamlfile = CustomerBuild.yamlbuilder(alldata, region_users)
-        yamlfile = CustomerBuild.yamlbuilder(alldata, region_users, user_data)
         yamlfile_as_json = yaml.safe_load(yamlfile)
         self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
 
@@ -350,8 +346,7 @@ class CreateResource(unittest.TestCase):
         ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-2'
         domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
         mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = True
-        user_data = {'user_domain': 'default', 'user_id': 'user1'}
+        yamlfile = CustomerBuild.yamlbuilder(alldata, region_users_v4)
-        yamlfile = CustomerBuild.yamlbuilder(alldata, region_users_v4, user_data)
         yamlfile_as_json = yaml.safe_load(yamlfile)
         self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
         self.assertEqual(yaml.safe_load(yamlfile), yaml.safe_load(fullyaml_aic4))

requirements.txt

@@ -16,4 +16,4 @@ keystonemiddleware>=4.17.0 # Apache-2.0
 python-keystoneclient>=3.8.0 # Apache-2.0
 mysqlclient>=1.4.0
 psutil>=3.2.2 # BSD
-setuptools>=40.0.0
+setuptools<50.0.0,>=40.0.0