Updated schema code & schema generator
This commit is contained in:
Maxim Kulkin
parent
849edbd652
commit
7c013ea912
@ -7,6 +7,8 @@ from rubick.discovery import OpenstackDiscovery
|
||||
import rubick.inspections
|
||||
# Silence PEP8 "unused import"
|
||||
assert rubick.inspections
|
||||
import rubick.schemas
|
||||
assert rubick.schemas
|
||||
from rubick.json import openstack_for_json
|
||||
|
||||
|
||||
|
||||
@ -1,2 +1,10 @@
|
||||
class ValidatorException(BaseException):
|
||||
class RubickException(BaseException):
|
||||
pass
|
||||
|
||||
|
||||
class ValidatorException(RubickException):
|
||||
pass
|
||||
|
||||
|
||||
class SchemaException(RubickException):
|
||||
pass
|
||||
|
||||
126
rubick/schema.py
126
rubick/schema.py
@ -1,98 +1,61 @@
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
|
||||
from rubick.common import Issue, MarkedIssue, Mark, Version, find, index
|
||||
from rubick.exceptions import RubickException
|
||||
|
||||
|
||||
class SchemaUpdateRecord(object):
|
||||
class SchemaError(RubickException):
|
||||
pass
|
||||
|
||||
|
||||
class SchemaVersionRecord(object):
|
||||
# checkpoint's data is version number
|
||||
def __init__(self, version, checkpoint):
|
||||
super(SchemaVersionRecord, self).__init__()
|
||||
|
||||
def __init__(self, version, operation, data=None):
|
||||
super(SchemaUpdateRecord, self).__init__()
|
||||
if not operation in ['checkpoint', 'add', 'remove']:
|
||||
raise Error('Unknown operation "%s"' % operation)
|
||||
version = Version(version)
|
||||
self.version = version
|
||||
self.operation = operation
|
||||
self.data = data
|
||||
self.version = Version(version)
|
||||
self.checkpoint = checkpoint
|
||||
|
||||
self.adds = []
|
||||
self.removals = []
|
||||
self._current_section = 'DEFAULT'
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
'<SchemaUpdateRecord %s %s %s' % (
|
||||
self.version,
|
||||
self.operation,
|
||||
self.data)
|
||||
'<SchemaVersionRecord %s%s>' % (
|
||||
self.version, ' (checkpoint)' if self.checkpoint else '')
|
||||
)
|
||||
|
||||
|
||||
class SchemaBuilder(object):
|
||||
|
||||
def __init__(self, name, data):
|
||||
super(SchemaBuilder, self).__init__()
|
||||
self.name = name
|
||||
self.data = data
|
||||
|
||||
self.current_version = None
|
||||
self.current_section = None
|
||||
self.adds = []
|
||||
self.removals = []
|
||||
|
||||
def __del__(self):
|
||||
if len(self.adds) > 0 or len(self.removals) > 0:
|
||||
sys.stderr.write(
|
||||
"WARNING: Uncommitted config schema \"%s\" version %s\n" %
|
||||
(self.name, self.current_version))
|
||||
|
||||
def version(self, version, checkpoint=False):
|
||||
version = Version(version)
|
||||
|
||||
if self.current_version and self.current_version != version:
|
||||
self.commit()
|
||||
|
||||
if checkpoint or self.data == []:
|
||||
self.data.append(SchemaUpdateRecord(version, 'checkpoint'))
|
||||
|
||||
self.current_version = version
|
||||
def __cmp__(self, other):
|
||||
return self.version.__cmp__(other.version)
|
||||
|
||||
def section(self, name):
|
||||
self.current_section = name
|
||||
self._current_section = name
|
||||
|
||||
def param(self, *args, **kwargs):
|
||||
self._ensure_version()
|
||||
|
||||
if not 'section' in kwargs and self.current_section:
|
||||
kwargs['section'] = self.current_section
|
||||
if not 'section' in kwargs and self._current_section:
|
||||
kwargs['section'] = self._current_section
|
||||
|
||||
self.adds.append(ConfigParameterSchema(*args, **kwargs))
|
||||
|
||||
def remove_param(self, name):
|
||||
self._ensure_version()
|
||||
|
||||
self.removals.append(name)
|
||||
|
||||
def commit(self):
|
||||
"Finalize schema building"
|
||||
self._ensure_version()
|
||||
|
||||
if len(self.removals) > 0:
|
||||
self.data.append(
|
||||
SchemaUpdateRecord(
|
||||
self.current_version,
|
||||
'remove',
|
||||
self.removals))
|
||||
self.removals = []
|
||||
if len(self.adds) > 0:
|
||||
self.data.append(
|
||||
SchemaUpdateRecord(
|
||||
self.current_version,
|
||||
'add',
|
||||
self.adds))
|
||||
self.adds = []
|
||||
class SchemaBuilder(object):
|
||||
|
||||
def _ensure_version(self):
|
||||
if not self.current_version:
|
||||
raise Error(
|
||||
'Schema version is not specified. Please call version() '
|
||||
'method first')
|
||||
def __init__(self, data):
|
||||
super(SchemaBuilder, self).__init__()
|
||||
self.data = data
|
||||
|
||||
@contextmanager
|
||||
def version(self, version, checkpoint=False):
|
||||
version_record = SchemaVersionRecord(version, checkpoint)
|
||||
|
||||
yield version_record
|
||||
|
||||
self.data.append(version_record)
|
||||
self.data.sort()
|
||||
|
||||
|
||||
class ConfigSchemaRegistry:
|
||||
@ -103,8 +66,9 @@ class ConfigSchemaRegistry:
|
||||
if not configname:
|
||||
configname = '%s.conf' % project
|
||||
fullname = '%s/%s' % (project, configname)
|
||||
if fullname not in self.__schemas:
|
||||
self.__schemas[fullname] = []
|
||||
return SchemaBuilder(fullname, self.__schemas[fullname])
|
||||
return SchemaBuilder(self.__schemas[fullname])
|
||||
|
||||
@classmethod
|
||||
def get_schema(self, project, version, configname=None):
|
||||
@ -119,7 +83,7 @@ class ConfigSchemaRegistry:
|
||||
records = self.__schemas[fullname]
|
||||
i = len(records) - 1
|
||||
# Find latest checkpoint prior given version
|
||||
while i >= 0 and not (records[i].operation == 'checkpoint'
|
||||
while i >= 0 and not (records[i].checkpoint
|
||||
and records[i].version <= version):
|
||||
i -= 1
|
||||
|
||||
@ -132,8 +96,7 @@ class ConfigSchemaRegistry:
|
||||
|
||||
while i < len(records) and records[i].version <= version:
|
||||
last_version = records[i].version
|
||||
if records[i].operation == 'add':
|
||||
for param in records[i].data:
|
||||
for param in records[i].adds:
|
||||
if param.name in seen_parameters:
|
||||
old_param_index = index(
|
||||
parameters,
|
||||
@ -143,8 +106,7 @@ class ConfigSchemaRegistry:
|
||||
else:
|
||||
parameters.append(param)
|
||||
seen_parameters.add(param.name)
|
||||
elif records[i].operation == 'remove':
|
||||
for param_name in records[i].data:
|
||||
for param_name in records[i].removals:
|
||||
param_index = index(
|
||||
parameters,
|
||||
lambda p: p.name == param_name)
|
||||
@ -228,10 +190,10 @@ class TypeValidatorRegistry:
|
||||
return self.__validators[name]
|
||||
|
||||
|
||||
class SchemaError(Issue):
|
||||
class SchemaIssue(Issue):
|
||||
|
||||
def __init__(self, message):
|
||||
super(SchemaError, self).__init__(Issue.ERROR, message)
|
||||
super(SchemaIssue, self).__init__(Issue.ERROR, message)
|
||||
|
||||
|
||||
class InvalidValueError(MarkedIssue):
|
||||
@ -485,7 +447,7 @@ def validate_port(s, min=1, max=65535):
|
||||
def validate_list(s, element_type='string'):
|
||||
element_type_validator = TypeValidatorRegistry.get_validator(element_type)
|
||||
if not element_type_validator:
|
||||
return SchemaError('Invalid element type "%s"' % element_type)
|
||||
return SchemaIssue('Invalid element type "%s"' % element_type)
|
||||
|
||||
result = []
|
||||
s = s.strip()
|
||||
@ -508,7 +470,7 @@ def validate_list(s, element_type='string'):
|
||||
def validate_dict(s, element_type='string'):
|
||||
element_type_validator = TypeValidatorRegistry.get_validator(element_type)
|
||||
if not element_type_validator:
|
||||
return SchemaError('Invalid element type "%s"' % element_type)
|
||||
return SchemaIssue('Invalid element type "%s"' % element_type)
|
||||
|
||||
result = {}
|
||||
s = s.strip()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1 +1,2 @@
|
||||
import rubick.schemas.keystone.v2013_1_3
|
||||
import rubick.schemas.keystone.v2013_1_4
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -2,115 +2,86 @@ from rubick.schema import ConfigSchemaRegistry
|
||||
|
||||
keystone = ConfigSchemaRegistry.register_schema(project='keystone')
|
||||
|
||||
keystone.version('2013.1.4')
|
||||
with keystone.version('2013.1.4') as keystone_2013_1_4:
|
||||
|
||||
keystone.section('DEFAULT')
|
||||
keystone_2013_1_4.section('ssl')
|
||||
|
||||
keystone.section('sql')
|
||||
keystone_2013_1_4.param('enable', type='boolean', default=True)
|
||||
|
||||
keystone.section('identity')
|
||||
keystone_2013_1_4.param('certfile', type='string',
|
||||
default='/etc/keystone/pki/certs/ssl_cert.pem')
|
||||
|
||||
keystone.section('credential')
|
||||
keystone_2013_1_4.param('keyfile', type='string',
|
||||
default='/etc/keystone/pki/private/ssl_key.pem')
|
||||
|
||||
keystone.section('trust')
|
||||
keystone_2013_1_4.param('ca_certs', type='string',
|
||||
default='/etc/keystone/pki/certs/cacert.pem')
|
||||
|
||||
keystone.section('os_inherit')
|
||||
keystone_2013_1_4.param('ca_key', type='string',
|
||||
default='/etc/keystone/pki/private/cakey.pem')
|
||||
|
||||
keystone.section('catalog')
|
||||
keystone_2013_1_4.param('key_size', type='integer', default=1024)
|
||||
|
||||
keystone.section('endpoint_filter')
|
||||
keystone_2013_1_4.param('valid_days', type='integer', default=3650)
|
||||
|
||||
keystone.section('token')
|
||||
keystone_2013_1_4.param('cert_required', type='boolean', default=False)
|
||||
|
||||
keystone.section('cache')
|
||||
keystone_2013_1_4.param('cert_subject', type='string',
|
||||
default='/CUS/STUnset/LUnset/OUnset/CNlocalhost')
|
||||
|
||||
keystone.section('policy')
|
||||
keystone_2013_1_4.section('signing')
|
||||
|
||||
keystone.section('ec2')
|
||||
|
||||
keystone.section('assignment')
|
||||
|
||||
keystone.section('oauth1')
|
||||
|
||||
keystone.section('ssl')
|
||||
|
||||
keystone.param('enable', type='string', default='True', description="")
|
||||
|
||||
keystone.param('certfile', type='string',
|
||||
default='/etc/keystone/pki/certs/ssl_cert.pem', description="")
|
||||
|
||||
keystone.param('keyfile', type='string',
|
||||
default='/etc/keystone/pki/private/ssl_key.pem', description="")
|
||||
|
||||
keystone.param('ca_certs', type='string',
|
||||
default='/etc/keystone/pki/certs/cacert.pem', description="")
|
||||
|
||||
keystone.param('ca_key', type='string',
|
||||
default='/etc/keystone/pki/private/cakey.pem', description="")
|
||||
|
||||
keystone.param('key_size', type='string', default='1024', description="")
|
||||
|
||||
keystone.param('valid_days', type='string', default='3650', description="")
|
||||
|
||||
keystone.param('cert_required', type='string', default='False', description="")
|
||||
|
||||
keystone.param('cert_subject', type='string',
|
||||
default='/CUS/STUnset/LUnset/OUnset/CNlocalhost',
|
||||
description="")
|
||||
|
||||
keystone.section('signing')
|
||||
|
||||
keystone.param('token_format', type='string', default='',
|
||||
keystone_2013_1_4.param(
|
||||
'token_format', type='string', default='',
|
||||
description="Deprecated in favor of provider in the [token] "
|
||||
"section Allowed values are PKI or UUID")
|
||||
|
||||
keystone.param('certfile', type='string',
|
||||
default='/etc/keystone/pki/certs/signing_cert.pem',
|
||||
description="")
|
||||
keystone_2013_1_4.param('certfile', type='string',
|
||||
default='/etc/keystone/pki/certs/signing_cert.pem')
|
||||
|
||||
keystone.param('keyfile', type='string',
|
||||
default='/etc/keystone/pki/private/signing_key.pem',
|
||||
description="")
|
||||
keystone_2013_1_4.param(
|
||||
'keyfile', type='string',
|
||||
default='/etc/keystone/pki/private/signing_key.pem')
|
||||
|
||||
keystone.param('ca_certs', type='string',
|
||||
default='/etc/keystone/pki/certs/cacert.pem', description="")
|
||||
keystone_2013_1_4.param('ca_certs', type='string',
|
||||
default='/etc/keystone/pki/certs/cacert.pem')
|
||||
|
||||
keystone.param('ca_key', type='string',
|
||||
default='/etc/keystone/pki/private/cakey.pem', description="")
|
||||
keystone_2013_1_4.param('ca_key', type='string',
|
||||
default='/etc/keystone/pki/private/cakey.pem')
|
||||
|
||||
keystone.param('key_size', type='string', default='2048', description="")
|
||||
keystone_2013_1_4.param('key_size', type='integer', default=2048)
|
||||
|
||||
keystone.param('valid_days', type='string', default='3650', description="")
|
||||
keystone_2013_1_4.param('valid_days', type='integer', default=3650)
|
||||
|
||||
keystone.param('cert_subject', type='string',
|
||||
default='/CUS/STUnset/LUnset/OUnset/CNwww.example.com',
|
||||
description="")
|
||||
keystone_2013_1_4.param(
|
||||
'cert_subject', type='string',
|
||||
default='/CUS/STUnset/LUnset/OUnset/CNwww.example.com')
|
||||
|
||||
keystone.section('ldap')
|
||||
keystone_2013_1_4.section('auth')
|
||||
|
||||
keystone.section('auth')
|
||||
keystone_2013_1_4.param('methods', type='string',
|
||||
default='external,password,token,oauth1')
|
||||
|
||||
keystone.param('methods', type='string',
|
||||
default='external,password,token,oauth1', description="")
|
||||
keystone_2013_1_4.param(
|
||||
'external', type='string',
|
||||
default='keystone_2013_1_4.auth.plugins.external.ExternalDefault')
|
||||
|
||||
keystone.param('external', type='string',
|
||||
default='keystone.auth.plugins.external.ExternalDefault',
|
||||
description="")
|
||||
keystone_2013_1_4.param(
|
||||
'password', type='string',
|
||||
default='keystone_2013_1_4.auth.plugins.password.Password')
|
||||
|
||||
keystone.param('password', type='string',
|
||||
default='keystone.auth.plugins.password.Password',
|
||||
description="")
|
||||
keystone_2013_1_4.param(
|
||||
'token', type='string',
|
||||
default='keystone_2013_1_4.auth.plugins.token.Token')
|
||||
|
||||
keystone.param('token', type='string',
|
||||
default='keystone.auth.plugins.token.Token', description="")
|
||||
keystone_2013_1_4.param(
|
||||
'oauth1', type='string',
|
||||
default='keystone_2013_1_4.auth.plugins.oauth1.OAuth')
|
||||
|
||||
keystone.param('oauth1', type='string',
|
||||
default='keystone.auth.plugins.oauth1.OAuth', description="")
|
||||
keystone_2013_1_4.section('paste_deploy')
|
||||
|
||||
keystone.section('paste_deploy')
|
||||
|
||||
keystone.param('config_file', type='string', default='keystone-paste.ini',
|
||||
keystone_2013_1_4.param(
|
||||
'config_file', type='string',
|
||||
default='keystone-paste.ini',
|
||||
description="Name of the paste configuration file that defines "
|
||||
"the available pipelines")
|
||||
|
||||
keystone.commit()
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
import rubick.schemas.nova.v2013_1
|
||||
import rubick.schemas.nova.v2013_1_3
|
||||
import rubick.schemas.nova.v2013_1_4
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -3,82 +3,109 @@ import re
|
||||
import sys
|
||||
|
||||
|
||||
class SchemaParser(object):
|
||||
|
||||
def parse_args(self, argv):
|
||||
def parse_args(argv):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--conf', dest='conf_file', default=None,
|
||||
help='Path to configuration file sample')
|
||||
parser.add_argument('--project_name', dest='prj_name', default='nova',
|
||||
help='Name of the configurations project')
|
||||
parser.add_argument('--config_version', dest='conf_ver',
|
||||
default='2013.1.3', help='Version of the package')
|
||||
parser.add_argument('project',
|
||||
help='Name of the project (e.g. "nova")')
|
||||
parser.add_argument('version',
|
||||
help='Version of the project (e.g. "2013.1.3")')
|
||||
parser.add_argument('config_file',
|
||||
help='Config file sample to process')
|
||||
args = parser.parse_args(argv[1:])
|
||||
return args
|
||||
|
||||
def generate_schema(self, file_to_open, file_to_generate='/tmp/sample.py'):
|
||||
with open(file_to_open, 'r') as f:
|
||||
content = f.readlines()
|
||||
with open(file_to_generate, 'w') as f:
|
||||
|
||||
def generate_schema(project, version, config_file, schema_file=None):
|
||||
if not schema_file:
|
||||
schema_file = '%s_%s.py' % (project, version.replace('.', '_'))
|
||||
|
||||
with open(config_file, 'r') as f:
|
||||
config_lines = f.readlines()
|
||||
|
||||
conf_variable = '%s_%s' % (project, version.replace('.', '_'))
|
||||
with open(schema_file, 'w') as f:
|
||||
f.write("""from rubick.schema import ConfigSchemaRegistry
|
||||
|
||||
%s = ConfigSchemaRegistry.register_schema(project='%s')
|
||||
{0} = ConfigSchemaRegistry.register_schema(project='{0}')
|
||||
|
||||
%s.version('%s')
|
||||
|
||||
""" % (self.prj_name, self.prj_name, self.prj_name, self.conf_ver)
|
||||
with {0}.version('{1}') as {2}:""".format(project, version, conf_variable)
|
||||
)
|
||||
for index, line in enumerate(content):
|
||||
if str(line).startswith('['):
|
||||
f.write("%s.section('%s')\n\n" % (
|
||||
self.prj_name, str(line).strip('[]\n')))
|
||||
continue
|
||||
if str(line).startswith('# ') or str(line).startswith(
|
||||
'\n') or str(line).startswith('#\n'):
|
||||
continue
|
||||
else:
|
||||
revers_list = content[0:index]
|
||||
revers_list.reverse()
|
||||
comments = []
|
||||
for comment in revers_list:
|
||||
if str(comment).startswith('# '):
|
||||
comments.append(comment)
|
||||
else:
|
||||
break
|
||||
comments.reverse()
|
||||
|
||||
comment_str = ''.join(comments).replace('#', '').replace(
|
||||
'\n', '').replace('\"', '\'').rstrip(' ').lstrip(' ')
|
||||
regex = re.search('^.*\((.*?) value.*$', comment_str)
|
||||
|
||||
if regex:
|
||||
var_type = regex.group(1)
|
||||
else:
|
||||
var_type = 'string'
|
||||
|
||||
comment_str = re.sub(r' \((.*?) value.*$', '', comment_str)
|
||||
|
||||
wrk_str = str(line).strip('#[]\n')
|
||||
f.write(
|
||||
"%s.param('%s', type='%s', "
|
||||
"default='%s', description=\"%s\")\n\n" % (
|
||||
self.prj_name,
|
||||
wrk_str.split('=')[0].rstrip(' ').lstrip(' '),
|
||||
var_type.rstrip(' ').lstrip(' '),
|
||||
''.join(wrk_str.split('=')[1:]).rstrip(' ').lstrip(
|
||||
' '),
|
||||
comment_str))
|
||||
description_lines = []
|
||||
for line in config_lines:
|
||||
if line.startswith('['):
|
||||
section_name = line.strip('[]\n')
|
||||
f.write("\n\n %s.section('%s')" % (
|
||||
conf_variable, section_name))
|
||||
description_lines = []
|
||||
continue
|
||||
|
||||
def run(self, argv):
|
||||
args = self.parse_args(argv)
|
||||
if line.strip() in ['', '#']:
|
||||
description_lines = []
|
||||
continue
|
||||
|
||||
if line.startswith('# '):
|
||||
description_lines.append(line[2:].strip())
|
||||
continue
|
||||
|
||||
description = ' '.join(description_lines)
|
||||
match = re.search('^(.*)\((.*?) value\)$', description)
|
||||
if match:
|
||||
description = match.group(1)
|
||||
param_type = match.group(2).strip()
|
||||
if param_type == 'floating point':
|
||||
param_type = 'float'
|
||||
else:
|
||||
param_type = 'string'
|
||||
|
||||
line = line.strip('#\n')
|
||||
param_name, param_value = [
|
||||
s.strip() for s in re.split('[:=]', line, 1)]
|
||||
|
||||
# Normalizing param value and type
|
||||
if param_value == '<None>':
|
||||
param_value = None
|
||||
elif param_type == 'boolean':
|
||||
if param_value.lower() == 'false':
|
||||
param_value = False
|
||||
elif param_value.lower() == 'true':
|
||||
param_value = True
|
||||
elif param_type == 'integer':
|
||||
param_value = int(param_value)
|
||||
if param_name.endswith('_port'):
|
||||
param_type = 'port'
|
||||
elif param_type == 'float':
|
||||
param_value = float(param_value)
|
||||
elif param_type == 'list':
|
||||
param_type = 'string_list'
|
||||
if param_value == '':
|
||||
param_value = []
|
||||
else:
|
||||
param_value = param_value.split(',')
|
||||
elif (param_type == 'string' and
|
||||
param_name.endswith('_host') and
|
||||
param_value in ['0.0.0.0', 'localhost', '127.0.0.1']):
|
||||
param_type = 'host'
|
||||
elif param_type == 'string' and param_name.endswith('_listen'):
|
||||
param_type = 'host'
|
||||
|
||||
f.write("\n\n %s.param('%s', type='%s', default=%s" % (
|
||||
conf_variable, param_name, param_type, repr(param_value)))
|
||||
f.write(", description=\"%s\"" % (
|
||||
description.replace('"', '\"')))
|
||||
f.write(")")
|
||||
|
||||
|
||||
def main(argv):
|
||||
args = parse_args(argv)
|
||||
params = vars(args)
|
||||
self.conf_file = params.pop('conf_file')
|
||||
self.prj_name = params.pop('prj_name')
|
||||
self.conf_ver = params.pop('conf_ver')
|
||||
self.generate_schema(self.conf_file)
|
||||
|
||||
project = params.pop('project')
|
||||
version = params.pop('version')
|
||||
config_file = params.pop('config_file')
|
||||
|
||||
generate_schema(project, version, config_file)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
runner = SchemaParser()
|
||||
runner.run(sys.argv)
|
||||
main(sys.argv)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user