Implemented advanced schema collector & generator (w/ code inspection)

requirements.txt

@@ -12,3 +12,4 @@ lettuce>=0.2.19
 pymongo==2.6.1
 recordtype==1.1
 paramiko==1.11.0
+oslo.config==1.2.1

rubick/schemas/schema_generator.py

@@ -1,6 +1,59 @@
 import argparse
 import re
 import sys
+import os
+import os.path
+import imp
+import traceback
+from copy import copy
+
+from oslo.config import cfg
+
+def identity(x):
+    return x
+
+__builtins__._ = identity
+
+
+class SchemaWriter(object):
+    def __init__(self, file, project, version):
+        super(SchemaWriter, self).__init__()
+        self.file = file
+        self.project = project
+        self.version = version
+        self._started = False
+        self._conf_variable = '%s_%s' % (self.project,
+                                         self.version.replace('.', '_'))
+
+    def _ensure_header(self):
+        if not self._started:
+            self._output_header()
+            self._started = True
+
+    def _output_header(self):
+        self.file.write("""from rubick.schema import ConfigSchemaRegistry
+
+{0} = ConfigSchemaRegistry.register_schema(project='{0}')
+
+with {0}.version('{1}') as {2}:""".format(self.project, self.version,
+                                          self._conf_variable))
+
+    def section(self, name):
+        self._ensure_header()
+        self.file.write("\n\n    %s.section('%s')" % (
+            self._conf_variable, name))
+
+    def param(self, name, type, default_value=None, description=None):
+        self._ensure_header()
+        self.file.write("\n\n    %s.param('%s', type='%s', default=%s" % (
+            self._conf_variable, name, type, repr(default_value)))
+        if description:
+            self.file.write(", description=\"%s\"" % (
+                description.replace('"', '\'')))
+        self.file.write(")")
+
+    def comment(self, text):
+        self.file.write("\n\n    # %s" % text)
 
 
 def parse_args(argv):
@@ -9,91 +62,266 @@ def parse_args(argv):
                         help='Name of the project (e.g. "nova")')
     parser.add_argument('version',
                         help='Version of the project (e.g. "2013.1.3")')
-    parser.add_argument('config_file',
+    parser.add_argument('config_or_module',
-                        help='Config file sample to process')
+                        help='Config file sample or Python module to process')
     args = parser.parse_args(argv[1:])
     return args
 
 
-def generate_schema(project, version, config_file, schema_file=None):
+def sanitize_type_and_value(param_name, param_type, param_value):
-    if not schema_file:
+    if param_value == '<None>':
-        schema_file = '%s_%s.py' % (project, version.replace('.', '_'))
+        param_value = None
+    elif param_type == 'boolean':
+        if param_value.lower() == 'false':
+            param_value = False
+        elif param_value.lower() == 'true':
+            param_value = True
+    elif param_type == 'integer':
+        param_value = int(param_value)
+        if param_name.endswith('_port'):
+            param_type = 'port'
+    elif param_type == 'float':
+        param_value = float(param_value)
+    elif param_type == 'list':
+        param_type = 'string_list'
+        if param_value == '':
+            param_value = []
+        else:
+            param_value = param_value.split(',')
+    elif (param_type == 'string' and
+            param_name.endswith('_host') and
+            param_value in ['0.0.0.0', 'localhost', '127.0.0.1']):
+        param_type = 'host'
+    elif param_type == 'string' and param_name.endswith('_listen'):
+        param_type = 'host'
 
+    return (param_type, param_value)
+
+
+def generate_schema_from_sample_config(project, version, config_file, schema_file=sys.stdout):
     with open(config_file, 'r') as f:
         config_lines = f.readlines()
 
-    conf_variable = '%s_%s' % (project, version.replace('.', '_'))
+    writer = SchemaWriter(schema_file, project, version)
-    with open(schema_file, 'w') as f:
-        f.write("""from rubick.schema import ConfigSchemaRegistry
 
-{0} = ConfigSchemaRegistry.register_schema(project='{0}')
+    description_lines = []
+    for line in config_lines:
+        if line.startswith('['):
+            section_name = line.strip('[]\n')
+            writer.section(section_name)
+            description_lines = []
+            continue
 
-with {0}.version('{1}') as {2}:""".format(project, version, conf_variable)
+        if line.strip() in ['', '#']:
-        )
+            description_lines = []
+            continue
 
-        description_lines = []
+        if line.startswith('# '):
-        for line in config_lines:
+            description_lines.append(line[2:].strip())
-            if line.startswith('['):
+            continue
-                section_name = line.strip('[]\n')
-                f.write("\n\n    %s.section('%s')" % (
-                    conf_variable, section_name))
-                description_lines = []
-                continue
 
-            if line.strip() in ['', '#']:
+        description = ' '.join(description_lines)
-                description_lines = []
+        match = re.search('^(.*)\((.*?) value\)$', description)
-                continue
+        if match:
+            description = match.group(1)
+            param_type = match.group(2).strip()
+            if param_type == 'floating point':
+                param_type = 'float'
+        else:
+            param_type = 'string'
 
-            if line.startswith('# '):
+        line = line.strip('#\n')
-                description_lines.append(line[2:].strip())
+        param_name, param_value = [
-                continue
+            s.strip() for s in re.split('[:=]', line, 1)]
 
-            description = ' '.join(description_lines)
+        (param_type, param_value) = \
-            match = re.search('^(.*)\((.*?) value\)$', description)
+            sanitize_type_and_value(param_name, param_type, param_value)
-            if match:
-                description = match.group(1)
-                param_type = match.group(2).strip()
-                if param_type == 'floating point':
-                    param_type = 'float'
-            else:
-                param_type = 'string'
 
-            line = line.strip('#\n')
+        writer.param(param_name, param_type, param_value, description)
-            param_name, param_value = [
-                s.strip() for s in re.split('[:=]', line, 1)]
 
-            # Normalizing param value and type
-            if param_value == '<None>':
-                param_value = None
-            elif param_type == 'boolean':
-                if param_value.lower() == 'false':
-                    param_value = False
-                elif param_value.lower() == 'true':
-                    param_value = True
-            elif param_type == 'integer':
-                param_value = int(param_value)
-                if param_name.endswith('_port'):
-                    param_type = 'port'
-            elif param_type == 'float':
-                param_value = float(param_value)
-            elif param_type == 'list':
-                param_type = 'string_list'
-                if param_value == '':
-                    param_value = []
-                else:
-                    param_value = param_value.split(',')
-            elif (param_type == 'string' and
-                  param_name.endswith('_host') and
-                  param_value in ['0.0.0.0', 'localhost', '127.0.0.1']):
-                param_type = 'host'
-            elif param_type == 'string' and param_name.endswith('_listen'):
-                param_type = 'host'
 
-            f.write("\n\n    %s.param('%s', type='%s', default=%s" % (
+OPT_TYPE_MAPPING = {
-                conf_variable, param_name, param_type, repr(param_value)))
+    'StrOpt': 'string',
-            f.write(", description=\"%s\"" % (
+    'BoolOpt': 'boolean',
-                description.replace('"', '\'')))
+    'IntOpt': 'integer',
-            f.write(")")
+    'FloatOpt': 'float',
+    'ListOpt': 'list',
+    'MultiStrOpt': 'multi'
+}
+
+
+OPTION_REGEX = re.compile(r"(%s)" % "|".join(OPT_TYPE_MAPPING.keys()))
+
+
+def generate_schema_from_code(project, version, module_path,
+                              schema_file=sys.stdout):
+    old_sys_path = copy(sys.path)
+
+    mods_by_pkg = dict()
+    filepaths = []
+    module_directory = ''
+
+    if os.path.isdir(module_path):
+        module_directory = module_path
+        while module_directory != '':
+            # TODO: handle .pyc and .pyo
+            if not os.path.isfile(
+                    os.path.join(module_directory, '__init__.py')):
+                break
+
+            module_directory = os.path.dirname(module_directory)
+
+        if not module_directory in sys.path:
+            sys.path.insert(0, module_directory)
+
+        for (dirpath, _, filenames) in os.walk(module_path):
+            for filename in filenames:
+                if not filename.endswith('.py'):
+                    continue
+
+                filepath = os.path.join(dirpath, filename)
+                with open(filepath) as f:
+                    content = f.read()
+                    if not re.search('Opt\(', content):
+                        continue
+
+                filepaths.append(filepath)
+    else:
+        filepaths.append(module_path)
+
+    for filepath in filepaths:
+        pkg_name = filepath.split(os.sep)[1]
+        mod_path = filepath
+        if module_directory != '':
+            mod_path = filepath.replace(module_directory + '/', '', 1)
+        mod_str = '.'.join(['.'.join(mod_path.split(os.sep)[:-1]),
+                           os.path.basename(mod_path).split('.')[0]])
+
+        mods_by_pkg.setdefault(pkg_name, list()).append(mod_str)
+
+    pkg_names = filter(lambda x: x.endswith('.py'), mods_by_pkg.keys())
+    pkg_names.sort()
+    ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys())
+    ext_names.sort()
+    pkg_names.extend(ext_names)
+
+    # opts_by_group is a mapping of group name to an options list
+    # The options list is a list of (module, options) tuples
+    opts_by_group = {'DEFAULT': []}
+
+    for pkg_name in pkg_names:
+        mods = mods_by_pkg.get(pkg_name)
+        mods.sort()
+        for mod_str in mods:
+            if mod_str.endswith('.__init__'):
+                mod_str = mod_str[:mod_str.rfind(".")]
+
+            mod_obj = _import_module(mod_str)
+            if not mod_obj:
+                raise RuntimeError("Unable to import module %s" % mod_str)
+
+            for group, opts in _list_opts(mod_obj):
+                opts_by_group.setdefault(group, []).append((mod_str, opts))
+
+    writer = SchemaWriter(schema_file, project, version)
+    print_group_opts(writer, 'DEFAULT', opts_by_group.pop('DEFAULT', []))
+    for group, opts in opts_by_group.items():
+        print_group_opts(writer, group, opts)
+
+    sys.path = old_sys_path
+
+
+def _import_module(mod_str):
+    try:
+        if mod_str.startswith('bin.'):
+            imp.load_source(mod_str[4:], os.path.join('bin', mod_str[4:]))
+            return sys.modules[mod_str[4:]]
+        else:
+            __import__(mod_str)
+            return sys.modules[mod_str]
+    except ImportError as ie:
+        traceback.print_exc()
+        # sys.stderr.write("%s\n" % str(ie))
+        return None
+    except Exception:
+        traceback.print_exc()
+        return None
+
+
+def _is_in_group(opt, group):
+    "Check if opt is in group."
+    for key, value in group._opts.items():
+        if value['opt'] == opt:
+            return True
+    return False
+
+
+def _guess_groups(opt, mod_obj):
+    # is it in the DEFAULT group?
+    if _is_in_group(opt, cfg.CONF):
+        return 'DEFAULT'
+
+    # what other groups is it in?
+    for key, value in cfg.CONF.items():
+        if not isinstance(value, cfg.CONF.GroupAttr):
+            continue
+
+        if _is_in_group(opt, value._group):
+            return value._group.name
+
+    # raise RuntimeError(
+    #     "Unable to find group for option %s, "
+    #     "maybe it's defined twice in the same group?"
+    #     % opt.name
+    # )
+
+    return 'DEFAULT'
+
+
+def _list_opts(obj):
+    def is_opt(o):
+        return (isinstance(o, cfg.Opt) and
+                not isinstance(o, cfg.SubCommandOpt))
+
+    opts = list()
+    for attr_str in dir(obj):
+        attr_obj = getattr(obj, attr_str)
+        if is_opt(attr_obj):
+            opts.append(attr_obj)
+        elif (isinstance(attr_obj, list) and
+              all(map(lambda x: is_opt(x), attr_obj))):
+            opts.extend(attr_obj)
+
+    ret = {}
+    for opt in opts:
+        ret.setdefault(_guess_groups(opt, obj), []).append(opt)
+    return ret.items()
+
+
+def print_group_opts(writer, group, opts_by_module):
+    writer.section(group)
+    for mod, opts in opts_by_module:
+        writer.comment("Options defined in %s" % mod)
+        for opt in opts:
+            print_opt(writer, opt)
+
+
+def print_opt(writer, opt):
+    opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help
+
+    if not opt_help:
+        sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name)
+        opt_help = ""
+
+    opt_type = None
+    try:
+        opt_type = OPT_TYPE_MAPPING.get(
+            OPTION_REGEX.search(str(type(opt))).group(0))
+    except (ValueError, AttributeError) as err:
+        sys.stderr.write("%s\n" % str(err))
+        opt_type = 'string'
+
+    writer.param(opt_name, opt_type, opt_default, opt_help)
 
 
 def main(argv):
@@ -102,9 +330,11 @@ def main(argv):
 
     project = params.pop('project')
     version = params.pop('version')
-    config_file = params.pop('config_file')
+    path = params.pop('config_or_module')
-
+    if os.path.isdir(path) or path.endswith('.py'):
-    generate_schema(project, version, config_file)
+        generate_schema_from_code(project, version, path)
+    else:
+        generate_schema_from_sample_config(project, version, path)
 
 
 if __name__ == '__main__':