Updated schema code & schema generator

discover_test.py

@@ -7,6 +7,8 @@ from rubick.discovery import OpenstackDiscovery
 import rubick.inspections
 # Silence PEP8 "unused import"
 assert rubick.inspections
+import rubick.schemas
+assert rubick.schemas
 from rubick.json import openstack_for_json
 
 

rubick/exceptions.py

@@ -1,2 +1,10 @@
-class ValidatorException(BaseException):
+class RubickException(BaseException):
+    pass
+
+
+class ValidatorException(RubickException):
+    pass
+
+
+class SchemaException(RubickException):
     pass

rubick/schema.py

@@ -1,98 +1,61 @@
-import sys
+from contextlib import contextmanager
 
 from rubick.common import Issue, MarkedIssue, Mark, Version, find, index
+from rubick.exceptions import RubickException
 
 
-class SchemaUpdateRecord(object):
+class SchemaError(RubickException):
+    pass
+
+
+class SchemaVersionRecord(object):
     # checkpoint's data is version number
+    def __init__(self, version, checkpoint):
+        super(SchemaVersionRecord, self).__init__()
 
-    def __init__(self, version, operation, data=None):
+        self.version = Version(version)
-        super(SchemaUpdateRecord, self).__init__()
+        self.checkpoint = checkpoint
-        if not operation in ['checkpoint', 'add', 'remove']:
+
-            raise Error('Unknown operation "%s"' % operation)
+        self.adds = []
-        version = Version(version)
+        self.removals = []
-        self.version = version
+        self._current_section = 'DEFAULT'
-        self.operation = operation
-        self.data = data
 
     def __repr__(self):
         return (
-            '<SchemaUpdateRecord %s %s %s' % (
+            '<SchemaVersionRecord %s%s>' % (
-                self.version,
+                self.version, ' (checkpoint)' if self.checkpoint else '')
-                self.operation,
-                self.data)
         )
 
-
+    def __cmp__(self, other):
-class SchemaBuilder(object):
+        return self.version.__cmp__(other.version)
-
-    def __init__(self, name, data):
-        super(SchemaBuilder, self).__init__()
-        self.name = name
-        self.data = data
-
-        self.current_version = None
-        self.current_section = None
-        self.adds = []
-        self.removals = []
-
-    def __del__(self):
-        if len(self.adds) > 0 or len(self.removals) > 0:
-            sys.stderr.write(
-                "WARNING: Uncommitted config schema \"%s\" version %s\n" %
-                (self.name, self.current_version))
-
-    def version(self, version, checkpoint=False):
-        version = Version(version)
-
-        if self.current_version and self.current_version != version:
-            self.commit()
-
-        if checkpoint or self.data == []:
-            self.data.append(SchemaUpdateRecord(version, 'checkpoint'))
-
-        self.current_version = version
 
     def section(self, name):
-        self.current_section = name
+        self._current_section = name
 
     def param(self, *args, **kwargs):
-        self._ensure_version()
+        if not 'section' in kwargs and self._current_section:
-
+            kwargs['section'] = self._current_section
-        if not 'section' in kwargs and self.current_section:
-            kwargs['section'] = self.current_section
 
         self.adds.append(ConfigParameterSchema(*args, **kwargs))
 
     def remove_param(self, name):
-        self._ensure_version()
-
         self.removals.append(name)
 
-    def commit(self):
-        "Finalize schema building"
-        self._ensure_version()
 
-        if len(self.removals) > 0:
+class SchemaBuilder(object):
-            self.data.append(
-                SchemaUpdateRecord(
-                    self.current_version,
-                    'remove',
-                    self.removals))
-            self.removals = []
-        if len(self.adds) > 0:
-            self.data.append(
-                SchemaUpdateRecord(
-                    self.current_version,
-                    'add',
-                    self.adds))
-            self.adds = []
 
-    def _ensure_version(self):
+    def __init__(self, data):
-        if not self.current_version:
+        super(SchemaBuilder, self).__init__()
-            raise Error(
+        self.data = data
-                'Schema version is not specified. Please call version() '
+
-                'method first')
+    @contextmanager
+    def version(self, version, checkpoint=False):
+        version_record = SchemaVersionRecord(version, checkpoint)
+
+        yield version_record
+
+        self.data.append(version_record)
+        self.data.sort()
 
 
 class ConfigSchemaRegistry:
@@ -103,8 +66,9 @@ class ConfigSchemaRegistry:
         if not configname:
             configname = '%s.conf' % project
         fullname = '%s/%s' % (project, configname)
-        self.__schemas[fullname] = []
+        if fullname not in self.__schemas:
-        return SchemaBuilder(fullname, self.__schemas[fullname])
+            self.__schemas[fullname] = []
+        return SchemaBuilder(self.__schemas[fullname])
 
     @classmethod
     def get_schema(self, project, version, configname=None):
@@ -119,7 +83,7 @@ class ConfigSchemaRegistry:
         records = self.__schemas[fullname]
         i = len(records) - 1
         # Find latest checkpoint prior given version
-        while i >= 0 and not (records[i].operation == 'checkpoint'
+        while i >= 0 and not (records[i].checkpoint
                               and records[i].version <= version):
             i -= 1
 
@@ -132,25 +96,23 @@ class ConfigSchemaRegistry:
 
         while i < len(records) and records[i].version <= version:
             last_version = records[i].version
-            if records[i].operation == 'add':
+            for param in records[i].adds:
-                for param in records[i].data:
+                if param.name in seen_parameters:
-                    if param.name in seen_parameters:
+                    old_param_index = index(
-                        old_param_index = index(
-                            parameters,
-                            lambda p: p.name == param.name)
-                        if old_param_index != -1:
-                            parameters[old_param_index] = param
-                    else:
-                        parameters.append(param)
-                        seen_parameters.add(param.name)
-            elif records[i].operation == 'remove':
-                for param_name in records[i].data:
-                    param_index = index(
                         parameters,
-                        lambda p: p.name == param_name)
+                        lambda p: p.name == param.name)
-                    if index != -1:
+                    if old_param_index != -1:
-                        parameters.pop(param_index)
+                        parameters[old_param_index] = param
-                        seen_parameters.remove(param_name)
+                else:
+                    parameters.append(param)
+                    seen_parameters.add(param.name)
+            for param_name in records[i].removals:
+                param_index = index(
+                    parameters,
+                    lambda p: p.name == param_name)
+                if index != -1:
+                    parameters.pop(param_index)
+                    seen_parameters.remove(param_name)
             i += 1
 
         return ConfigSchema(fullname, last_version, 'ini', parameters)
@@ -207,12 +169,12 @@ class ConfigParameterSchema:
         return (
             '<ConfigParameterSchema %s>' % ' '.join(
                 ['%s=%s' % (attr, getattr(self, attr))
-                                                    for attr in ['section',
+                    for attr in ['section',
-                                                                 'name',
+                                 'name',
-                                                                 'type',
+                                 'type',
-                                                                 'description',
+                                 'description',
-                                                                 'default',
+                                 'default',
-                                                                 'required']])
+                                 'required']])
         )
 
 
@@ -228,10 +190,10 @@ class TypeValidatorRegistry:
         return self.__validators[name]
 
 
-class SchemaError(Issue):
+class SchemaIssue(Issue):
 
     def __init__(self, message):
-        super(SchemaError, self).__init__(Issue.ERROR, message)
+        super(SchemaIssue, self).__init__(Issue.ERROR, message)
 
 
 class InvalidValueError(MarkedIssue):
@@ -438,7 +400,7 @@ def validate_string(s):
 def validate_integer(s, min=None, max=None):
     leading_whitespace_len = 0
     while leading_whitespace_len < len(s) \
-        and s[leading_whitespace_len].isspace():
+            and s[leading_whitespace_len].isspace():
         leading_whitespace_len += 1
 
     s = s.strip()
@@ -485,7 +447,7 @@ def validate_port(s, min=1, max=65535):
 def validate_list(s, element_type='string'):
     element_type_validator = TypeValidatorRegistry.get_validator(element_type)
     if not element_type_validator:
-        return SchemaError('Invalid element type "%s"' % element_type)
+        return SchemaIssue('Invalid element type "%s"' % element_type)
 
     result = []
     s = s.strip()
@@ -508,7 +470,7 @@ def validate_list(s, element_type='string'):
 def validate_dict(s, element_type='string'):
     element_type_validator = TypeValidatorRegistry.get_validator(element_type)
     if not element_type_validator:
-        return SchemaError('Invalid element type "%s"' % element_type)
+        return SchemaIssue('Invalid element type "%s"' % element_type)
 
     result = {}
     s = s.strip()

rubick/schemas/keystone/__init__.py

@@ -1 +1,2 @@
 import rubick.schemas.keystone.v2013_1_3
+import rubick.schemas.keystone.v2013_1_4

rubick/schemas/keystone/v2013_1_4.py

@@ -2,115 +2,86 @@ from rubick.schema import ConfigSchemaRegistry
 
 keystone = ConfigSchemaRegistry.register_schema(project='keystone')
 
-keystone.version('2013.1.4')
+with keystone.version('2013.1.4') as keystone_2013_1_4:
 
-keystone.section('DEFAULT')
+    keystone_2013_1_4.section('ssl')
 
-keystone.section('sql')
+    keystone_2013_1_4.param('enable', type='boolean', default=True)
 
-keystone.section('identity')
+    keystone_2013_1_4.param('certfile', type='string',
+                            default='/etc/keystone/pki/certs/ssl_cert.pem')
 
-keystone.section('credential')
+    keystone_2013_1_4.param('keyfile', type='string',
+                            default='/etc/keystone/pki/private/ssl_key.pem')
 
-keystone.section('trust')
+    keystone_2013_1_4.param('ca_certs', type='string',
+                            default='/etc/keystone/pki/certs/cacert.pem')
 
-keystone.section('os_inherit')
+    keystone_2013_1_4.param('ca_key', type='string',
+                            default='/etc/keystone/pki/private/cakey.pem')
 
-keystone.section('catalog')
+    keystone_2013_1_4.param('key_size', type='integer', default=1024)
 
-keystone.section('endpoint_filter')
+    keystone_2013_1_4.param('valid_days', type='integer', default=3650)
 
-keystone.section('token')
+    keystone_2013_1_4.param('cert_required', type='boolean', default=False)
 
-keystone.section('cache')
+    keystone_2013_1_4.param('cert_subject', type='string',
+                            default='/CUS/STUnset/LUnset/OUnset/CNlocalhost')
 
-keystone.section('policy')
+    keystone_2013_1_4.section('signing')
 
-keystone.section('ec2')
+    keystone_2013_1_4.param(
+        'token_format', type='string', default='',
+        description="Deprecated in favor of provider in the [token] "
+        "section Allowed values are PKI or UUID")
 
-keystone.section('assignment')
+    keystone_2013_1_4.param('certfile', type='string',
+                            default='/etc/keystone/pki/certs/signing_cert.pem')
 
-keystone.section('oauth1')
+    keystone_2013_1_4.param(
+        'keyfile', type='string',
+        default='/etc/keystone/pki/private/signing_key.pem')
 
-keystone.section('ssl')
+    keystone_2013_1_4.param('ca_certs', type='string',
+                            default='/etc/keystone/pki/certs/cacert.pem')
 
-keystone.param('enable', type='string', default='True', description="")
+    keystone_2013_1_4.param('ca_key', type='string',
+                            default='/etc/keystone/pki/private/cakey.pem')
 
-keystone.param('certfile', type='string',
+    keystone_2013_1_4.param('key_size', type='integer', default=2048)
-               default='/etc/keystone/pki/certs/ssl_cert.pem', description="")
 
-keystone.param('keyfile', type='string',
+    keystone_2013_1_4.param('valid_days', type='integer', default=3650)
-               default='/etc/keystone/pki/private/ssl_key.pem', description="")
 
-keystone.param('ca_certs', type='string',
+    keystone_2013_1_4.param(
-               default='/etc/keystone/pki/certs/cacert.pem', description="")
+        'cert_subject', type='string',
+        default='/CUS/STUnset/LUnset/OUnset/CNwww.example.com')
 
-keystone.param('ca_key', type='string',
+    keystone_2013_1_4.section('auth')
-               default='/etc/keystone/pki/private/cakey.pem', description="")
 
-keystone.param('key_size', type='string', default='1024', description="")
+    keystone_2013_1_4.param('methods', type='string',
+                            default='external,password,token,oauth1')
 
-keystone.param('valid_days', type='string', default='3650', description="")
+    keystone_2013_1_4.param(
+        'external', type='string',
+        default='keystone_2013_1_4.auth.plugins.external.ExternalDefault')
 
-keystone.param('cert_required', type='string', default='False', description="")
+    keystone_2013_1_4.param(
+        'password', type='string',
+        default='keystone_2013_1_4.auth.plugins.password.Password')
 
-keystone.param('cert_subject', type='string',
+    keystone_2013_1_4.param(
-               default='/CUS/STUnset/LUnset/OUnset/CNlocalhost',
+        'token', type='string',
-               description="")
+        default='keystone_2013_1_4.auth.plugins.token.Token')
 
-keystone.section('signing')
+    keystone_2013_1_4.param(
+        'oauth1', type='string',
+        default='keystone_2013_1_4.auth.plugins.oauth1.OAuth')
 
-keystone.param('token_format', type='string', default='',
+    keystone_2013_1_4.section('paste_deploy')
-               description="Deprecated in favor of provider in the [token] "
-                           "section Allowed values are PKI or UUID")
 
-keystone.param('certfile', type='string',
+    keystone_2013_1_4.param(
-               default='/etc/keystone/pki/certs/signing_cert.pem',
+        'config_file', type='string',
-               description="")
+        default='keystone-paste.ini',
-
+        description="Name of the paste configuration file that defines "
-keystone.param('keyfile', type='string',
+        "the available pipelines")
-               default='/etc/keystone/pki/private/signing_key.pem',
-               description="")
-
-keystone.param('ca_certs', type='string',
-               default='/etc/keystone/pki/certs/cacert.pem', description="")
-
-keystone.param('ca_key', type='string',
-               default='/etc/keystone/pki/private/cakey.pem', description="")
-
-keystone.param('key_size', type='string', default='2048', description="")
-
-keystone.param('valid_days', type='string', default='3650', description="")
-
-keystone.param('cert_subject', type='string',
-               default='/CUS/STUnset/LUnset/OUnset/CNwww.example.com',
-               description="")
-
-keystone.section('ldap')
-
-keystone.section('auth')
-
-keystone.param('methods', type='string',
-               default='external,password,token,oauth1', description="")
-
-keystone.param('external', type='string',
-               default='keystone.auth.plugins.external.ExternalDefault',
-               description="")
-
-keystone.param('password', type='string',
-               default='keystone.auth.plugins.password.Password',
-               description="")
-
-keystone.param('token', type='string',
-               default='keystone.auth.plugins.token.Token', description="")
-
-keystone.param('oauth1', type='string',
-               default='keystone.auth.plugins.oauth1.OAuth', description="")
-
-keystone.section('paste_deploy')
-
-keystone.param('config_file', type='string', default='keystone-paste.ini',
-               description="Name of the paste configuration file that defines "
-                           "the available pipelines")
-
-keystone.commit()

rubick/schemas/nova/__init__.py

@@ -1,2 +1,2 @@
-import rubick.schemas.nova.v2013_1
 import rubick.schemas.nova.v2013_1_3
+import rubick.schemas.nova.v2013_1_4

rubick/schemas/schema_generator.py

@@ -3,82 +3,109 @@ import re
 import sys
 
 
-class SchemaParser(object):
+def parse_args(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('project',
+                        help='Name of the project (e.g. "nova")')
+    parser.add_argument('version',
+                        help='Version of the project (e.g. "2013.1.3")')
+    parser.add_argument('config_file',
+                        help='Config file sample to process')
+    args = parser.parse_args(argv[1:])
+    return args
 
-    def parse_args(self, argv):
-        parser = argparse.ArgumentParser()
-        parser.add_argument('--conf', dest='conf_file', default=None,
-                            help='Path to configuration file sample')
-        parser.add_argument('--project_name', dest='prj_name', default='nova',
-                            help='Name of the configurations project')
-        parser.add_argument('--config_version', dest='conf_ver',
-                            default='2013.1.3', help='Version of the package')
-        args = parser.parse_args(argv[1:])
-        return args
 
-    def generate_schema(self, file_to_open, file_to_generate='/tmp/sample.py'):
+def generate_schema(project, version, config_file, schema_file=None):
-        with open(file_to_open, 'r') as f:
+    if not schema_file:
-            content = f.readlines()
+        schema_file = '%s_%s.py' % (project, version.replace('.', '_'))
-        with open(file_to_generate, 'w') as f:
-            f.write("""from rubick.schema import ConfigSchemaRegistry
 
-%s = ConfigSchemaRegistry.register_schema(project='%s')
+    with open(config_file, 'r') as f:
+        config_lines = f.readlines()
 
-%s.version('%s')
+    conf_variable = '%s_%s' % (project, version.replace('.', '_'))
+    with open(schema_file, 'w') as f:
+        f.write("""from rubick.schema import ConfigSchemaRegistry
 
-""" % (self.prj_name, self.prj_name, self.prj_name, self.conf_ver)
+{0} = ConfigSchemaRegistry.register_schema(project='{0}')
-            )
+
-            for index, line in enumerate(content):
+with {0}.version('{1}') as {2}:""".format(project, version, conf_variable)
-                if str(line).startswith('['):
+        )
-                    f.write("%s.section('%s')\n\n" % (
+
-                        self.prj_name, str(line).strip('[]\n')))
+        description_lines = []
-                    continue
+        for line in config_lines:
-                if str(line).startswith('# ') or str(line).startswith(
+            if line.startswith('['):
-                        '\n') or str(line).startswith('#\n'):
+                section_name = line.strip('[]\n')
-                    continue
+                f.write("\n\n    %s.section('%s')" % (
+                    conf_variable, section_name))
+                description_lines = []
+                continue
+
+            if line.strip() in ['', '#']:
+                description_lines = []
+                continue
+
+            if line.startswith('# '):
+                description_lines.append(line[2:].strip())
+                continue
+
+            description = ' '.join(description_lines)
+            match = re.search('^(.*)\((.*?) value\)$', description)
+            if match:
+                description = match.group(1)
+                param_type = match.group(2).strip()
+                if param_type == 'floating point':
+                    param_type = 'float'
+            else:
+                param_type = 'string'
+
+            line = line.strip('#\n')
+            param_name, param_value = [
+                s.strip() for s in re.split('[:=]', line, 1)]
+
+            # Normalizing param value and type
+            if param_value == '<None>':
+                param_value = None
+            elif param_type == 'boolean':
+                if param_value.lower() == 'false':
+                    param_value = False
+                elif param_value.lower() == 'true':
+                    param_value = True
+            elif param_type == 'integer':
+                param_value = int(param_value)
+                if param_name.endswith('_port'):
+                    param_type = 'port'
+            elif param_type == 'float':
+                param_value = float(param_value)
+            elif param_type == 'list':
+                param_type = 'string_list'
+                if param_value == '':
+                    param_value = []
                 else:
-                    revers_list = content[0:index]
+                    param_value = param_value.split(',')
-                    revers_list.reverse()
+            elif (param_type == 'string' and
-                    comments = []
+                  param_name.endswith('_host') and
-                    for comment in revers_list:
+                  param_value in ['0.0.0.0', 'localhost', '127.0.0.1']):
-                        if str(comment).startswith('# '):
+                param_type = 'host'
-                            comments.append(comment)
+            elif param_type == 'string' and param_name.endswith('_listen'):
-                        else:
+                param_type = 'host'
-                            break
-                    comments.reverse()
 
-                    comment_str = ''.join(comments).replace('#', '').replace(
+            f.write("\n\n    %s.param('%s', type='%s', default=%s" % (
-                        '\n', '').replace('\"', '\'').rstrip(' ').lstrip(' ')
+                conf_variable, param_name, param_type, repr(param_value)))
-                    regex = re.search('^.*\((.*?) value.*$', comment_str)
+            f.write(", description=\"%s\"" % (
+                description.replace('"', '\"')))
+            f.write(")")
 
-                    if regex:
-                        var_type = regex.group(1)
-                    else:
-                        var_type = 'string'
 
-                    comment_str = re.sub(r' \((.*?) value.*$', '', comment_str)
+def main(argv):
+    args = parse_args(argv)
+    params = vars(args)
 
-                    wrk_str = str(line).strip('#[]\n')
+    project = params.pop('project')
-                    f.write(
+    version = params.pop('version')
-                        "%s.param('%s', type='%s', "
+    config_file = params.pop('config_file')
-                        "default='%s', description=\"%s\")\n\n" % (
-                            self.prj_name,
-                            wrk_str.split('=')[0].rstrip(' ').lstrip(' '),
-                            var_type.rstrip(' ').lstrip(' '),
-                            ''.join(wrk_str.split('=')[1:]).rstrip(' ').lstrip(
-                                ' '),
-                            comment_str))
-                    continue
 
-    def run(self, argv):
+    generate_schema(project, version, config_file)
-        args = self.parse_args(argv)
-        params = vars(args)
-        self.conf_file = params.pop('conf_file')
-        self.prj_name = params.pop('prj_name')
-        self.conf_ver = params.pop('conf_ver')
-        self.generate_schema(self.conf_file)
 
 
 if __name__ == '__main__':
-    runner = SchemaParser()
+    main(sys.argv)
-    runner.run(sys.argv)