x/ranger
Code Issues Proposed changes
ranger/orm/base_config.py
Gage Hugo 68a500b3de Pep8 and Ranger Image Build process update 
Updated pep8 requirements and fixed resulting
codes to fix broken pep8 job.

Updated docker image build process to use roles
to ensure docker rather than using manual docker
install process to fix broken ranger image job.

Co-Authored-By: Jeremy Houser <jeremyhouser@protonmail.com>
Co-Authored-By: Chi Lo <cl566n@att.com>

Change-Id: I28df0a27e4b354dd53c17fbb1a9468cb7ff5bc16
2020-09-24 15:26:26 +00:00

493 lines
16 KiB
Python
Raw Blame History

# Copyright 2018 AT&T Corporation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from os.path import join
from oslo_config import cfg
from configparser import ConfigParser
CONF = cfg.CONF
# Orm config options in DEFAULT block
OrmOpts = [
cfg.StrOpt('protocol',
default='http',
help='Orm protocol used.'),
cfg.HostAddressOpt('orm_host',
default='127.0.0.1',
help='Orm server IP address.'),
cfg.URIOpt('ranger_url',
default='http://127.0.0.1',
help='Ranger URL.'),
cfg.StrOpt('ranger_base',
default='/opt/stack/ranger',
help='Orm base directory.'),
cfg.BoolOpt('ssl_verify',
default=False,
help='Flag for SSL verfiy Enabled/Disabled.'),
cfg.BoolOpt('ranger_agent_https_enabled',
default=False,
help='Ranger Agent https_enabled flag - True/False.'),
cfg.StrOpt('ranger_agent_client_cert_path',
default='',
help='Ranger Agent certificate path.'),
cfg.IntOpt('resource_status_max_interval',
default=2,
help='Interval in seconds.'),
cfg.StrOpt('log_location',
default='/var/log/ranger',
help='Orm log directory.'),
cfg.StrOpt('debug_level',
default='ERROR',
choices=('CRITICAL', 'DEBUG', 'ERROR', 'INFO', 'WARNING'),
help='logging debug level'),
cfg.StrOpt('use_handlers',
choices=('console', 'logfile', 'console,logfile'),
default='console',
help='output logging to console or logfile or both'),
cfg.BoolOpt('pecan_debug',
default=True,
help='Service app log debug level Enabled/Disabled.'),
]
CONF.register_opts(OrmOpts)
# Keystone config options in [keystone_autotoken] group
orm_token_group = cfg.OptGroup(name='keystone_authtoken',
title='Orm Keystone Token Options')
OrmAuthGroup = [
cfg.StrOpt('username',
default='admin',
help='Token user name.'),
cfg.StrOpt('password',
default='xxxxxxxx',
help='Token password.'),
cfg.StrOpt('project_name',
default='admin',
help='Project name.'),
cfg.StrOpt('region',
default='RegionOne',
help='Region.'),
cfg.BoolOpt('auth_enabled',
default=False,
help='Auth token Enabled/Disabled.'),
cfg.StrOpt('auth_version',
default='v3',
help='Keystone version number.'),
cfg.StrOpt('project_domain_name',
default='default',
help='Project domain name.'),
cfg.StrOpt('user_domain_name',
default='default',
help='User domain name.'),
cfg.StrOpt('user_role',
default='admin',
help='token user role.')
]
CONF.register_group(orm_token_group)
CONF.register_opts(OrmAuthGroup, orm_token_group)
# Database config options in [database] group
orm_database_group = cfg.OptGroup(name='database',
title='Orm Database Options')
OrmDatabaseGroup = [
cfg.StrOpt('connection',
help='The SQLAlchemy connection string to use to connect to '
'the ORM database.',
secret=True),
cfg.IntOpt('max_retries',
default=-1,
help='The maximum number of retries for database connection.')
]
CONF.register_group(orm_database_group)
CONF.register_opts(OrmDatabaseGroup, orm_database_group)
# UUID config options in [uuid] group
orm_uuid_group = cfg.OptGroup(name='uuid', title='Orm UUID Options')
OrmUuidGroup = [
cfg.PortOpt('port',
default=7001,
help='UUID port.'),
cfg.StrOpt('url',
default='http://127.0.0.1:7001/',
help='UUID url.'),
cfg.StrOpt('log',
default='uuidgen.log',
help='UUID log name.')
]
CONF.register_group(orm_uuid_group)
CONF.register_opts(OrmUuidGroup, orm_uuid_group)
# cms config options in [cms] group
orm_cms_group = cfg.OptGroup(name='cms', title='Orm Cms Options')
OrmCmsGroup = [
cfg.PortOpt('port',
default=7080,
help='Cms port.'),
cfg.StrOpt('url',
default='http://127.0.0.1:7080/',
help='Cms url.'),
cfg.StrOpt('log',
default='cms.log',
help='Cms log name.')
]
CONF.register_group(orm_cms_group)
CONF.register_opts(OrmCmsGroup, orm_cms_group)
config = ConfigParser()
if CONF.config_file:
cfgfile = CONF.config_file[0]
else:
workspace = os.environ['PWD']
cfgfile = join(workspace, 'etc', 'ranger', 'ranger.conf')
config.read(cfgfile)
default_flavor = 'xx'
# fms config options in [fms] group
orm_fms_group = cfg.OptGroup(name='fms', title='Orm Fms Options')
OrmFmsGroup = [
cfg.PortOpt('port',
default=8082,
help='Fms port.'),
cfg.StrOpt('url',
default='http://127.0.0.1:8082/',
help='Fms url.'),
cfg.StrOpt('log',
default='fms.log',
help='Fms log name.'),
cfg.ListOpt('flavor_series',
default=[default_flavor],
help='Supported flavor series.'),
cfg.IntOpt('flavor_swap_file_limit',
default=0,
help='Flavor series swap file size limit.'),
cfg.IntOpt('flavor_ephemeral_limit',
default=0,
help='Flavor series ephemeral limit.')
]
CONF.register_group(orm_fms_group)
CONF.register_opts(OrmFmsGroup, orm_fms_group)
autogen_es = set()
flavor_group = cfg.OptGroup(name='flavor_series_metadata',
title='A Group of Flavor Series')
CONF.register_group(flavor_group)
if default_flavor in CONF.fms.flavor_series:
default_dict = {'alt_vcpu_limit': 1, 'alt_vram_limit': 1024}
FlavorGroup = [
cfg.DictOpt(default_flavor,
default=default_dict,
help="Dict that contains default flavor series metadata.")
]
CONF.register_opts(FlavorGroup, flavor_group)
else:
sections = config.sections()
series_metadata = [s for s in sections if "_flavor_series_metadata" in s]
for metadata in series_metadata:
flavor = metadata.replace('_flavor_series_metadata', '')
flavor_dict = dict(set(config.items(metadata))
- set(config.items('DEFAULT')))
FlavorGroup = [
cfg.DictOpt(flavor,
default=flavor_dict,
help="Dict that contains flavor series metadata. ")
]
CONF.register_opts(FlavorGroup, flavor_group)
for key, value in list(flavor_dict.items()):
if key.startswith("es_"):
autogen_es.add(value.split(': ')[0])
AutogenEsGroup = [
cfg.ListOpt('autogen_extra_specs',
default=list(autogen_es),
help="List of auto generated extra specs.")
]
CONF.register_opts(AutogenEsGroup, flavor_group)
# audit config options in [audit] group
orm_audit_group = cfg.OptGroup(name='audit', title='Orm Audit Options')
OrmAuditGroup = [
cfg.PortOpt('port',
default=7008,
help='Audit port.'),
cfg.StrOpt('url',
default='http://127.0.0.1:7008/',
help='Audit url.'),
cfg.StrOpt('log',
default='audit_server.log',
help='Audit log name.')
]
CONF.register_group(orm_audit_group)
CONF.register_opts(OrmAuditGroup, orm_audit_group)
# ims config options in [ims] group
orm_ims_group = cfg.OptGroup(name='ims', title='Orm Ims Options')
OrmImsGroup = [
cfg.PortOpt('port',
default=8084,
help='Ims port.'),
cfg.StrOpt('url',
default='http://127.0.0.1:8084/',
help='Ims url.'),
cfg.StrOpt('log',
default='ims.log',
help='Ims log name.')
]
CONF.register_group(orm_ims_group)
CONF.register_opts(OrmImsGroup, orm_ims_group)
# rms config options in [rms] group
# old 7003
orm_rms_group = cfg.OptGroup(name='rms', title='Orm Rms Options')
OrmRmsGroup = [
cfg.PortOpt('port',
default=7003,
help='Rms port.'),
cfg.StrOpt('url',
default='http://127.0.0.1:7003',
help='Rms url.'),
cfg.StrOpt('log',
default='rms.log',
help='Rms log name.')
]
CONF.register_group(orm_rms_group)
CONF.register_opts(OrmRmsGroup, orm_rms_group)
# rds config options in [rds] group
orm_rds_group = cfg.OptGroup(name='rds', title='Orm Rds Options')
OrmRdsGroup = [
cfg.PortOpt('port',
default=8777,
help='Rds port.'),
cfg.StrOpt('url',
default='http://127.0.0.1:8777/',
help='Rds url.'),
cfg.StrOpt('customer_domain',
default='default',
help='Keystone project domain'),
cfg.StrOpt('log',
default='rds.log',
help='Rds log name.')
]
CONF.register_group(orm_rds_group)
CONF.register_opts(OrmRdsGroup, orm_rds_group)
# cli config options in [cli] group
orm_cli_group = cfg.OptGroup(name='cli', title='Orm CLI Options')
OrmCliGroup = [
cfg.StrOpt('base_region',
default='RegionOne',
help='Base region.')
]
CONF.register_group(orm_cli_group)
CONF.register_opts(OrmCliGroup, orm_cli_group)
################################################################
# The following global variables are kept in here for
# backward compatiblitiy as other modules still
# referenced to them.
################################################################
# logging config settings
debug_level = CONF.debug_level
pecan_debug = CONF.pecan_debug
handler_list = CONF.use_handlers.split(",")
# ranger settings
protocol = CONF.protocol
orm_host = CONF.orm_host
ranger_url = CONF.ranger_url
ranger_base = CONF.ranger_base
user_domain_name = CONF.keystone_authtoken.user_domain_name
project_domain_name = CONF.keystone_authtoken.project_domain_name
conn = CONF.database.connection
db_connect = conn.replace("mysql+pymysql", "mysql") if conn else None
ssl_verify = CONF.ssl_verify
token_auth_version = '3' if (CONF.keystone_authtoken.auth_version
== 'v3') else '2.0'
cert_path = CONF.ranger_agent_client_cert_path
https_enabled = CONF.ranger_agent_https_enabled
uuid = {'port': CONF.uuid.port,
'base_url': CONF.uuid.url,
'log': '{}/{}'.format(CONF.log_location, CONF.uuid.log)}
cms = {'port': CONF.cms.port,
'base_url': CONF.cms.url,
'log': '{}/{}'.format(CONF.log_location, CONF.cms.log)}
fms = {'port': CONF.fms.port,
'base_url': CONF.fms.url,
'log': '{}/{}'.format(CONF.log_location, CONF.fms.log)}
audit_server = {'port': CONF.audit.port,
'base_url': CONF.audit.url,
'log': '{}/{}'.format(CONF.log_location, CONF.audit.log)}
ims = {'port': CONF.ims.port,
'base_url': CONF.ims.url,
'log': '{}/{}'.format(CONF.log_location, CONF.ims.log)}
# Had to remove trailing slash from port number in base_url so rms would work.
rms = {'port': CONF.rms.port,
'base_url': CONF.rms.url,
'log': '{}/{}'.format(CONF.log_location, CONF.rms.log)}
rds = {'port': CONF.rds.port,
'token_version': CONF.keystone_authtoken.auth_version,
'customer_domain': CONF.rds.customer_domain,
'base_url': CONF.rds.url,
'log': '{}/{}'.format(CONF.log_location, CONF.rds.log)}
cli = {'base_region': CONF.cli.base_region}
def server_request_auth(service_name):
services = ['cms', 'fms', 'ims', 'rms', 'uuid']
if service_name in services:
policy_name = CONF.ranger_base + '/etc/policy.json'
else:
# policy_N/A for services 'rds', and 'audit'
policy_name = None
# authentication settings
request_authentication = {
"enabled": CONF.keystone_authtoken.auth_enabled,
# The Keystone version currently in use. For Ranger,
# use '3' by default.
"keystone_version": token_auth_version,
"mech_id": CONF.keystone_authtoken.username,
"mech_pass": CONF.keystone_authtoken.password,
"policy_file": policy_name,
# The Keystone collection under which the role was granted.
# The key can be either "tenant" (for Keystone v2.0) or "domain"
# (for Keystone v3) and the value is the tenant/domain name.
"role_location": {"domain": "admin"},
"tenant_name": CONF.keystone_authtoken.project_name,
"rms_url": rms['base_url'],
"project_domain_name": CONF.keystone_authtoken.project_domain_name,
"token_role": CONF.keystone_authtoken.user_role,
"user_domain_name": CONF.keystone_authtoken.user_domain_name
}
return request_authentication
def get_log_config(log_file_name, ranger_service, ranger_service_module):
# Ranger logging template - we want to have the option of not
# routing to logfiles for all loggers except 'pecan' and
# 'py.warnings', which only logs to console
logging_template = {
'root': {'level': 'INFO', 'handlers': handler_list},
'loggers': {
ranger_service_module: {
'level': debug_level,
'handlers': handler_list,
'propagate': False
},
'orm.common.orm_common': {
'level': debug_level,
'handlers': handler_list,
'propagate': False
},
'orm.common.client.keystone.keystone_utils': {
'level': debug_level,
'handlers': handler_list,
'propagate': False
},
'orm.common.client.audit.audit_client': {
'level': debug_level,
'handlers': handler_list,
'propagate': False
},
'pecan': {'level': debug_level, 'handlers': handler_list,
'propagate': False},
'py.warnings': {'handlers': handler_list},
'__force_dict__': True
},
'handlers': {
'console': {
'level': debug_level,
'class': 'logging.StreamHandler',
'formatter': 'color'
},
'logfile': {
'level': debug_level,
'class': 'logging.handlers.RotatingFileHandler',
'maxBytes': 50000000,
'backupCount': 10,
'filename': log_file_name,
'formatter': 'simple'
}
},
'formatters': {
'simple': {
'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
'[%(threadName)s] %(message)s')
},
'color': {
'()': 'pecan.log.ColorFormatter',
'format': (
'%(asctime)s [%(padded_color_levelname)s] [%(name)s]'
'[%(threadName)s] %(message)s'),
'__force_dict__': True
}
}
}
service_logging = logging_template
if CONF.use_handlers == 'console':
del service_logging['handlers']['logfile']
elif CONF.use_handlers == 'Logfile':
del service_logging['handlers']['console']
return service_logging
View Git Blame Copy Permalink