Updated schema code & schema generator

This commit is contained in:
Maxim Kulkin 2013-10-25 16:30:06 +04:00
parent 849edbd652
commit 7c013ea912
12 changed files with 9981 additions and 13285 deletions

View File

@ -7,6 +7,8 @@ from rubick.discovery import OpenstackDiscovery
import rubick.inspections
# Silence PEP8 "unused import"
assert rubick.inspections
import rubick.schemas
assert rubick.schemas
from rubick.json import openstack_for_json

View File

@ -1,2 +1,10 @@
class ValidatorException(BaseException):
class RubickException(BaseException):
pass
class ValidatorException(RubickException):
pass
class SchemaException(RubickException):
pass

View File

@ -1,98 +1,61 @@
import sys
from contextlib import contextmanager
from rubick.common import Issue, MarkedIssue, Mark, Version, find, index
from rubick.exceptions import RubickException
class SchemaUpdateRecord(object):
class SchemaError(RubickException):
pass
class SchemaVersionRecord(object):
# checkpoint's data is version number
def __init__(self, version, checkpoint):
super(SchemaVersionRecord, self).__init__()
def __init__(self, version, operation, data=None):
super(SchemaUpdateRecord, self).__init__()
if not operation in ['checkpoint', 'add', 'remove']:
raise Error('Unknown operation "%s"' % operation)
version = Version(version)
self.version = version
self.operation = operation
self.data = data
self.version = Version(version)
self.checkpoint = checkpoint
self.adds = []
self.removals = []
self._current_section = 'DEFAULT'
def __repr__(self):
return (
'<SchemaUpdateRecord %s %s %s' % (
self.version,
self.operation,
self.data)
'<SchemaVersionRecord %s%s>' % (
self.version, ' (checkpoint)' if self.checkpoint else '')
)
class SchemaBuilder(object):
def __init__(self, name, data):
super(SchemaBuilder, self).__init__()
self.name = name
self.data = data
self.current_version = None
self.current_section = None
self.adds = []
self.removals = []
def __del__(self):
if len(self.adds) > 0 or len(self.removals) > 0:
sys.stderr.write(
"WARNING: Uncommitted config schema \"%s\" version %s\n" %
(self.name, self.current_version))
def version(self, version, checkpoint=False):
version = Version(version)
if self.current_version and self.current_version != version:
self.commit()
if checkpoint or self.data == []:
self.data.append(SchemaUpdateRecord(version, 'checkpoint'))
self.current_version = version
def __cmp__(self, other):
return self.version.__cmp__(other.version)
def section(self, name):
self.current_section = name
self._current_section = name
def param(self, *args, **kwargs):
self._ensure_version()
if not 'section' in kwargs and self.current_section:
kwargs['section'] = self.current_section
if not 'section' in kwargs and self._current_section:
kwargs['section'] = self._current_section
self.adds.append(ConfigParameterSchema(*args, **kwargs))
def remove_param(self, name):
self._ensure_version()
self.removals.append(name)
def commit(self):
"Finalize schema building"
self._ensure_version()
if len(self.removals) > 0:
self.data.append(
SchemaUpdateRecord(
self.current_version,
'remove',
self.removals))
self.removals = []
if len(self.adds) > 0:
self.data.append(
SchemaUpdateRecord(
self.current_version,
'add',
self.adds))
self.adds = []
class SchemaBuilder(object):
def _ensure_version(self):
if not self.current_version:
raise Error(
'Schema version is not specified. Please call version() '
'method first')
def __init__(self, data):
super(SchemaBuilder, self).__init__()
self.data = data
@contextmanager
def version(self, version, checkpoint=False):
version_record = SchemaVersionRecord(version, checkpoint)
yield version_record
self.data.append(version_record)
self.data.sort()
class ConfigSchemaRegistry:
@ -103,8 +66,9 @@ class ConfigSchemaRegistry:
if not configname:
configname = '%s.conf' % project
fullname = '%s/%s' % (project, configname)
self.__schemas[fullname] = []
return SchemaBuilder(fullname, self.__schemas[fullname])
if fullname not in self.__schemas:
self.__schemas[fullname] = []
return SchemaBuilder(self.__schemas[fullname])
@classmethod
def get_schema(self, project, version, configname=None):
@ -119,7 +83,7 @@ class ConfigSchemaRegistry:
records = self.__schemas[fullname]
i = len(records) - 1
# Find latest checkpoint prior given version
while i >= 0 and not (records[i].operation == 'checkpoint'
while i >= 0 and not (records[i].checkpoint
and records[i].version <= version):
i -= 1
@ -132,25 +96,23 @@ class ConfigSchemaRegistry:
while i < len(records) and records[i].version <= version:
last_version = records[i].version
if records[i].operation == 'add':
for param in records[i].data:
if param.name in seen_parameters:
old_param_index = index(
parameters,
lambda p: p.name == param.name)
if old_param_index != -1:
parameters[old_param_index] = param
else:
parameters.append(param)
seen_parameters.add(param.name)
elif records[i].operation == 'remove':
for param_name in records[i].data:
param_index = index(
for param in records[i].adds:
if param.name in seen_parameters:
old_param_index = index(
parameters,
lambda p: p.name == param_name)
if index != -1:
parameters.pop(param_index)
seen_parameters.remove(param_name)
lambda p: p.name == param.name)
if old_param_index != -1:
parameters[old_param_index] = param
else:
parameters.append(param)
seen_parameters.add(param.name)
for param_name in records[i].removals:
param_index = index(
parameters,
lambda p: p.name == param_name)
if index != -1:
parameters.pop(param_index)
seen_parameters.remove(param_name)
i += 1
return ConfigSchema(fullname, last_version, 'ini', parameters)
@ -207,12 +169,12 @@ class ConfigParameterSchema:
return (
'<ConfigParameterSchema %s>' % ' '.join(
['%s=%s' % (attr, getattr(self, attr))
for attr in ['section',
'name',
'type',
'description',
'default',
'required']])
for attr in ['section',
'name',
'type',
'description',
'default',
'required']])
)
@ -228,10 +190,10 @@ class TypeValidatorRegistry:
return self.__validators[name]
class SchemaError(Issue):
class SchemaIssue(Issue):
def __init__(self, message):
super(SchemaError, self).__init__(Issue.ERROR, message)
super(SchemaIssue, self).__init__(Issue.ERROR, message)
class InvalidValueError(MarkedIssue):
@ -438,7 +400,7 @@ def validate_string(s):
def validate_integer(s, min=None, max=None):
leading_whitespace_len = 0
while leading_whitespace_len < len(s) \
and s[leading_whitespace_len].isspace():
and s[leading_whitespace_len].isspace():
leading_whitespace_len += 1
s = s.strip()
@ -485,7 +447,7 @@ def validate_port(s, min=1, max=65535):
def validate_list(s, element_type='string'):
element_type_validator = TypeValidatorRegistry.get_validator(element_type)
if not element_type_validator:
return SchemaError('Invalid element type "%s"' % element_type)
return SchemaIssue('Invalid element type "%s"' % element_type)
result = []
s = s.strip()
@ -508,7 +470,7 @@ def validate_list(s, element_type='string'):
def validate_dict(s, element_type='string'):
element_type_validator = TypeValidatorRegistry.get_validator(element_type)
if not element_type_validator:
return SchemaError('Invalid element type "%s"' % element_type)
return SchemaIssue('Invalid element type "%s"' % element_type)
result = {}
s = s.strip()

File diff suppressed because it is too large Load Diff

View File

@ -1 +1,2 @@
import rubick.schemas.keystone.v2013_1_3
import rubick.schemas.keystone.v2013_1_4

File diff suppressed because it is too large Load Diff

View File

@ -2,115 +2,86 @@ from rubick.schema import ConfigSchemaRegistry
keystone = ConfigSchemaRegistry.register_schema(project='keystone')
keystone.version('2013.1.4')
with keystone.version('2013.1.4') as keystone_2013_1_4:
keystone.section('DEFAULT')
keystone_2013_1_4.section('ssl')
keystone.section('sql')
keystone_2013_1_4.param('enable', type='boolean', default=True)
keystone.section('identity')
keystone_2013_1_4.param('certfile', type='string',
default='/etc/keystone/pki/certs/ssl_cert.pem')
keystone.section('credential')
keystone_2013_1_4.param('keyfile', type='string',
default='/etc/keystone/pki/private/ssl_key.pem')
keystone.section('trust')
keystone_2013_1_4.param('ca_certs', type='string',
default='/etc/keystone/pki/certs/cacert.pem')
keystone.section('os_inherit')
keystone_2013_1_4.param('ca_key', type='string',
default='/etc/keystone/pki/private/cakey.pem')
keystone.section('catalog')
keystone_2013_1_4.param('key_size', type='integer', default=1024)
keystone.section('endpoint_filter')
keystone_2013_1_4.param('valid_days', type='integer', default=3650)
keystone.section('token')
keystone_2013_1_4.param('cert_required', type='boolean', default=False)
keystone.section('cache')
keystone_2013_1_4.param('cert_subject', type='string',
default='/CUS/STUnset/LUnset/OUnset/CNlocalhost')
keystone.section('policy')
keystone_2013_1_4.section('signing')
keystone.section('ec2')
keystone_2013_1_4.param(
'token_format', type='string', default='',
description="Deprecated in favor of provider in the [token] "
"section Allowed values are PKI or UUID")
keystone.section('assignment')
keystone_2013_1_4.param('certfile', type='string',
default='/etc/keystone/pki/certs/signing_cert.pem')
keystone.section('oauth1')
keystone_2013_1_4.param(
'keyfile', type='string',
default='/etc/keystone/pki/private/signing_key.pem')
keystone.section('ssl')
keystone_2013_1_4.param('ca_certs', type='string',
default='/etc/keystone/pki/certs/cacert.pem')
keystone.param('enable', type='string', default='True', description="")
keystone_2013_1_4.param('ca_key', type='string',
default='/etc/keystone/pki/private/cakey.pem')
keystone.param('certfile', type='string',
default='/etc/keystone/pki/certs/ssl_cert.pem', description="")
keystone_2013_1_4.param('key_size', type='integer', default=2048)
keystone.param('keyfile', type='string',
default='/etc/keystone/pki/private/ssl_key.pem', description="")
keystone_2013_1_4.param('valid_days', type='integer', default=3650)
keystone.param('ca_certs', type='string',
default='/etc/keystone/pki/certs/cacert.pem', description="")
keystone_2013_1_4.param(
'cert_subject', type='string',
default='/CUS/STUnset/LUnset/OUnset/CNwww.example.com')
keystone.param('ca_key', type='string',
default='/etc/keystone/pki/private/cakey.pem', description="")
keystone_2013_1_4.section('auth')
keystone.param('key_size', type='string', default='1024', description="")
keystone_2013_1_4.param('methods', type='string',
default='external,password,token,oauth1')
keystone.param('valid_days', type='string', default='3650', description="")
keystone_2013_1_4.param(
'external', type='string',
default='keystone_2013_1_4.auth.plugins.external.ExternalDefault')
keystone.param('cert_required', type='string', default='False', description="")
keystone_2013_1_4.param(
'password', type='string',
default='keystone_2013_1_4.auth.plugins.password.Password')
keystone.param('cert_subject', type='string',
default='/CUS/STUnset/LUnset/OUnset/CNlocalhost',
description="")
keystone_2013_1_4.param(
'token', type='string',
default='keystone_2013_1_4.auth.plugins.token.Token')
keystone.section('signing')
keystone_2013_1_4.param(
'oauth1', type='string',
default='keystone_2013_1_4.auth.plugins.oauth1.OAuth')
keystone.param('token_format', type='string', default='',
description="Deprecated in favor of provider in the [token] "
"section Allowed values are PKI or UUID")
keystone_2013_1_4.section('paste_deploy')
keystone.param('certfile', type='string',
default='/etc/keystone/pki/certs/signing_cert.pem',
description="")
keystone.param('keyfile', type='string',
default='/etc/keystone/pki/private/signing_key.pem',
description="")
keystone.param('ca_certs', type='string',
default='/etc/keystone/pki/certs/cacert.pem', description="")
keystone.param('ca_key', type='string',
default='/etc/keystone/pki/private/cakey.pem', description="")
keystone.param('key_size', type='string', default='2048', description="")
keystone.param('valid_days', type='string', default='3650', description="")
keystone.param('cert_subject', type='string',
default='/CUS/STUnset/LUnset/OUnset/CNwww.example.com',
description="")
keystone.section('ldap')
keystone.section('auth')
keystone.param('methods', type='string',
default='external,password,token,oauth1', description="")
keystone.param('external', type='string',
default='keystone.auth.plugins.external.ExternalDefault',
description="")
keystone.param('password', type='string',
default='keystone.auth.plugins.password.Password',
description="")
keystone.param('token', type='string',
default='keystone.auth.plugins.token.Token', description="")
keystone.param('oauth1', type='string',
default='keystone.auth.plugins.oauth1.OAuth', description="")
keystone.section('paste_deploy')
keystone.param('config_file', type='string', default='keystone-paste.ini',
description="Name of the paste configuration file that defines "
"the available pipelines")
keystone.commit()
keystone_2013_1_4.param(
'config_file', type='string',
default='keystone-paste.ini',
description="Name of the paste configuration file that defines "
"the available pipelines")

View File

@ -1,2 +1,2 @@
import rubick.schemas.nova.v2013_1
import rubick.schemas.nova.v2013_1_3
import rubick.schemas.nova.v2013_1_4

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -3,82 +3,109 @@ import re
import sys
class SchemaParser(object):
def parse_args(argv):
parser = argparse.ArgumentParser()
parser.add_argument('project',
help='Name of the project (e.g. "nova")')
parser.add_argument('version',
help='Version of the project (e.g. "2013.1.3")')
parser.add_argument('config_file',
help='Config file sample to process')
args = parser.parse_args(argv[1:])
return args
def parse_args(self, argv):
parser = argparse.ArgumentParser()
parser.add_argument('--conf', dest='conf_file', default=None,
help='Path to configuration file sample')
parser.add_argument('--project_name', dest='prj_name', default='nova',
help='Name of the configurations project')
parser.add_argument('--config_version', dest='conf_ver',
default='2013.1.3', help='Version of the package')
args = parser.parse_args(argv[1:])
return args
def generate_schema(self, file_to_open, file_to_generate='/tmp/sample.py'):
with open(file_to_open, 'r') as f:
content = f.readlines()
with open(file_to_generate, 'w') as f:
f.write("""from rubick.schema import ConfigSchemaRegistry
def generate_schema(project, version, config_file, schema_file=None):
if not schema_file:
schema_file = '%s_%s.py' % (project, version.replace('.', '_'))
%s = ConfigSchemaRegistry.register_schema(project='%s')
with open(config_file, 'r') as f:
config_lines = f.readlines()
%s.version('%s')
conf_variable = '%s_%s' % (project, version.replace('.', '_'))
with open(schema_file, 'w') as f:
f.write("""from rubick.schema import ConfigSchemaRegistry
""" % (self.prj_name, self.prj_name, self.prj_name, self.conf_ver)
)
for index, line in enumerate(content):
if str(line).startswith('['):
f.write("%s.section('%s')\n\n" % (
self.prj_name, str(line).strip('[]\n')))
continue
if str(line).startswith('# ') or str(line).startswith(
'\n') or str(line).startswith('#\n'):
continue
{0} = ConfigSchemaRegistry.register_schema(project='{0}')
with {0}.version('{1}') as {2}:""".format(project, version, conf_variable)
)
description_lines = []
for line in config_lines:
if line.startswith('['):
section_name = line.strip('[]\n')
f.write("\n\n %s.section('%s')" % (
conf_variable, section_name))
description_lines = []
continue
if line.strip() in ['', '#']:
description_lines = []
continue
if line.startswith('# '):
description_lines.append(line[2:].strip())
continue
description = ' '.join(description_lines)
match = re.search('^(.*)\((.*?) value\)$', description)
if match:
description = match.group(1)
param_type = match.group(2).strip()
if param_type == 'floating point':
param_type = 'float'
else:
param_type = 'string'
line = line.strip('#\n')
param_name, param_value = [
s.strip() for s in re.split('[:=]', line, 1)]
# Normalizing param value and type
if param_value == '<None>':
param_value = None
elif param_type == 'boolean':
if param_value.lower() == 'false':
param_value = False
elif param_value.lower() == 'true':
param_value = True
elif param_type == 'integer':
param_value = int(param_value)
if param_name.endswith('_port'):
param_type = 'port'
elif param_type == 'float':
param_value = float(param_value)
elif param_type == 'list':
param_type = 'string_list'
if param_value == '':
param_value = []
else:
revers_list = content[0:index]
revers_list.reverse()
comments = []
for comment in revers_list:
if str(comment).startswith('# '):
comments.append(comment)
else:
break
comments.reverse()
param_value = param_value.split(',')
elif (param_type == 'string' and
param_name.endswith('_host') and
param_value in ['0.0.0.0', 'localhost', '127.0.0.1']):
param_type = 'host'
elif param_type == 'string' and param_name.endswith('_listen'):
param_type = 'host'
comment_str = ''.join(comments).replace('#', '').replace(
'\n', '').replace('\"', '\'').rstrip(' ').lstrip(' ')
regex = re.search('^.*\((.*?) value.*$', comment_str)
f.write("\n\n %s.param('%s', type='%s', default=%s" % (
conf_variable, param_name, param_type, repr(param_value)))
f.write(", description=\"%s\"" % (
description.replace('"', '\"')))
f.write(")")
if regex:
var_type = regex.group(1)
else:
var_type = 'string'
comment_str = re.sub(r' \((.*?) value.*$', '', comment_str)
def main(argv):
args = parse_args(argv)
params = vars(args)
wrk_str = str(line).strip('#[]\n')
f.write(
"%s.param('%s', type='%s', "
"default='%s', description=\"%s\")\n\n" % (
self.prj_name,
wrk_str.split('=')[0].rstrip(' ').lstrip(' '),
var_type.rstrip(' ').lstrip(' '),
''.join(wrk_str.split('=')[1:]).rstrip(' ').lstrip(
' '),
comment_str))
continue
project = params.pop('project')
version = params.pop('version')
config_file = params.pop('config_file')
def run(self, argv):
args = self.parse_args(argv)
params = vars(args)
self.conf_file = params.pop('conf_file')
self.prj_name = params.pop('prj_name')
self.conf_ver = params.pop('conf_ver')
self.generate_schema(self.conf_file)
generate_schema(project, version, config_file)
if __name__ == '__main__':
runner = SchemaParser()
runner.run(sys.argv)
main(sys.argv)