Updated model, implemented model parser, added config schema inspection (WIP)

ostack_validator/common.py

@@ -11,6 +11,55 @@ def index(l, predicate):
     i += 1
   return -1
 
+class Version:
+  def __init__(self, major, minor=0, maintenance=0):
+    "Create Version object by either passing 3 integers, one string or an another Version object"
+    if isinstance(major, str):
+      self.parts = [int(x) for x in major.split('.', 3)]
+    elif isinstance(major, Version):
+      self.parts = major.parts
+    else:
+      self.parts = [int(major), int(minor), int(maintenance)]
+
+  @property
+  def major(self):
+    return self.parts[0]
+
+  @major.setter
+  def major(self, value):
+    self.parts[0] = int(value)
+
+  @property
+  def minor(self):
+    return self.parts[1]
+
+  @minor.setter
+  def minor(self, value):
+    self.parts[1] = int(value)
+
+  @property
+  def maintenance(self):
+    return self.parts[2]
+
+  @maintenance.setter
+  def maintenance(self, value):
+    self.parts[2] = value
+
+  def __str__(self):
+    return '.'.join([str(p) for p in self.parts])
+
+  def __repr__(self):
+    return '<Version %s>' % str(self)
+
+  def __cmp__(self, other):
+    for i in xrange(0, 3):
+      x = self.parts[i] - other.parts[i]
+      if x != 0:
+        return -1 if x < 0 else 1
+
+    return 0
+
+
 class Mark(object):
   def __init__(self, source, line, column):
     self.source = source

ostack_validator/inspection.py

@@ -0,0 +1,37 @@
+from common import Error, MarkedError, Mark
+from model import *
+
+import unittest
+
+from ostack_validator.common import Inspection
+from ostack_validator.schema import ConfigSchemaRegistry
+
+class MainConfigValidationInspection(Inspection):
+  def inspect(self, openstack):
+    results = []
+    for host in openstack.hosts:
+      for component in host.components:
+        main_config = component.get_config()
+
+        if not main_config:
+          results.append(Error('Missing main configuration file for component "%s" at host "%s"' % (component.name, host.name)))
+          continue
+
+        schema = ConfigSchemaRegistry.get_schema(component.name, component.version, main_config.name)
+        if not schema: continue
+
+        for parameter in main_config.parameters:
+          parameter_schema = schema.get_parameter(name=parameter.name, section=parameter.section)
+          # TBD: should we report unknown config parameters?
+          if not parameter_schema: continue
+
+          type_descriptor = TypeDescriptorRepository.get_type(parameter_schema.type)
+          type_validation_result = type_descriptor.validate(parameter.value)
+          if type_validation_result:
+            results.append(type_validation_result)
+
+    return results
+
+if __name__ == '__main__':
+  unittest.main()
+

ostack_validator/main.py

@@ -0,0 +1,26 @@
+import sys
+
+from ostack_validator.model_parser import ModelParser
+from ostack_validator.inspection import MainConfigValidationInspection
+
+def main(args):
+  if len(args) < 1:
+    print("Usage: validator <config-snapshot-path>")
+    sys.exit(1)
+
+  model_parser = ModelParser()
+
+  model = model_parser.parse(args[0])
+
+  inspections = [MainConfigValidationInspection()]
+
+  results = []
+  for inspection in inspections:
+    results.extend(inspection.inspect(model))
+
+  for result in results:
+    print(result)
+
+if __name__ == '__main__':
+  main(sys.argv[1:])
+

ostack_validator/model.py

@@ -1,20 +1,59 @@
 
 class Openstack(object):
-  def __init__(self, components):
+  def __init__(self, hosts, resource_locator, config_parser):
     super(Openstack, self).__init__()
+    self.hosts = hosts
+    self.resource_locator = resource_locator
+    self.config_parser = config_parser
+    for host in self.hosts:
+      host.parent = self
+
+class Host(object):
+  def __init__(self, name, metadata, components):
+    super(Host, self).__init__()
+    self.name = name
+    self.metadata = metadata
     self.components = components
+    for component in self.components:
+      component.parent = self
 
 class OpenstackComponent(object):
-  def __init__(self, name, version, configs=[]):
+  def __init__(self, name, version):
     super(OpenstackComponent, self).__init__()
     self.name = name
     self.version = version
+    self.configs = {}
+
+  @property
+  def host(self):
+    return self.parent
+
+  @property
+  def openstack(self):
+    return self.host.parent
+
+  def get_config(self, config_name=None):
+    if config_name is None:
+      config_name = '%s.conf' % self.name
+
+    if not config_name in self.configs:
+      resource = self.openstack.resource_locator.find_resource(self.host.name, self.name, config_name)
+      if resource:
+        config = self.openstack.config_parser.parse(config_name, resource.get_contents())
+        self.configs[config_name] = config
+      else:
+        self.configs[config_name] = None
+
+    return self.configs[config_name]
 
 class ComponentConfig(object):
   def __init__(self, name, sections=[], errors=[]):
     super(ComponentConfig, self).__init__()
     self.name = name
     self.sections = sections
+    for section in self.sections:
+      section.parent = self
+
     self.errors = errors
 
 class Element(object):
@@ -38,6 +77,8 @@ class ConfigSection(Element):
     super(ConfigSection, self).__init__(start_mark, end_mark)
     self.name = name
     self.parameters = parameters
+    for parameter in self.parameters:
+      parameter.parent = self
 
 class ConfigSectionName(TextElement): pass
 
@@ -45,8 +86,13 @@ class ConfigParameter(Element):
   def __init__(self, start_mark, end_mark, name, value, delimiter):
     super(ConfigParameter, self).__init__(start_mark, end_mark)
     self.name = name
+    self.name.parent = self
+
     self.value = value
+    self.value.parent = self
+
     self.delimiter = delimiter
+    self.delimiter.parent = self
 
   def __eq__(self, other):
     return (self.name.text == other.name.text) and (self.value.text == other.value.text)

ostack_validator/model_parser.py

@@ -0,0 +1,34 @@
+import logging
+
+from ostack_validator.common import Version
+from ostack_validator.model import *
+from ostack_validator.resource import ConfigSnapshotResourceLocator
+from ostack_validator.config_formats import IniConfigParser
+
+OPENSTACK_COMPONENTS = ['nova', 'keystone', 'glance']
+
+class ModelParser(object):
+  logger = logging.getLogger('ostack_validator.ModelParser')
+
+  def parse(self, path):
+    resource_locator = ConfigSnapshotResourceLocator(path)
+
+    hosts = []
+    for host_name in resource_locator.find_hosts():
+      components = []
+      for component_name in resource_locator.find_host_components(host_name):
+        if not component_name in OPENSTACK_COMPONENTS:
+          self.logger.warn('Unknown component in config: %s', component_name)
+          continue
+
+        component_version = Version(1000000) # very latest version
+        version_resource = resource_locator.find_resource(host_name, component_name, 'version')
+        if version_resource:
+          component_version = Version(version_resource.get_contents())
+
+        components.append(OpenstackComponent(component_name, component_version))
+
+      hosts.append(Host(host_name, {}, components))
+
+    return Openstack(hosts, resource_locator, IniConfigParser())
+

ostack_validator/resource.py

@@ -0,0 +1,56 @@
+import glob
+import os.path
+
+class Resource(object):
+  def __init__(self, name):
+    super(Resource, self).__init__()
+    self.name = name
+
+  def get_contents(self):
+    raise Error, 'Not implemented'
+
+class ResourceLocator(object):
+  def find_hosts(self):
+    return []
+
+  def find_host_components(self, host):
+    return []
+
+  def find_resource(self, host, component, name):
+    return None
+
+class FileResource(Resource):
+  def __init__(self, name, path):
+    super(FileResource, self).__init__(name)
+    self.path = path
+
+  def get_contents(self):
+    with open(self.path) as f:
+      return f.read()
+
+class ConfigSnapshotResourceLocator(object):
+  def __init__(self, basedir):
+    super(ConfigSnapshotResourceLocator, self).__init__()
+    self.basedir = basedir
+    if not os.path.isdir(self.basedir):
+      raise Error, 'Invalid argument: base directory does not exist'
+
+  def find_hosts(self):
+    return [os.path.basename(host_path) for host_path in glob.glob(os.path.join(self.basedir, '*')) if os.path.isdir(host_path)]
+
+  def find_host_components(self, host):
+    return [os.path.basename(component_path) for component_path in glob.glob(os.path.join(self.basedir, host, '*')) if os.path.isdir(component_path)]
+
+  def find_resource(self, host, component, name):
+    if not host:
+      raise Error, 'Invalid argument: "host" need to be specified'
+
+    if not component:
+      raise Error, 'Invalid argument: "component" need to be specified'
+
+    path = os.path.join(self.basedir, host, component, name)
+    if not os.path.exists(path):
+      return None
+
+    return FileResource(name, path)
+

ostack_validator/schema.py

@@ -1,52 +1,4 @@
-from ostack_validator.common import Inspection, MarkedError, Mark, find, index
-
-class Version:
-  def __init__(self, major, minor=0, maintenance=0):
-    "Create Version object by either passing 3 integers, one string or an another Version object"
-    if isinstance(major, str):
-      self.parts = [int(x) for x in major.split('.', 3)]
-    elif isinstance(major, Version):
-      self.parts = major.parts
-    else:
-      self.parts = [int(major), int(minor), int(maintenance)]
-
-  @property
-  def major(self):
-    return self.parts[0]
-
-  @major.setter
-  def major(self, value):
-    self.parts[0] = int(value)
-
-  @property
-  def minor(self):
-    return self.parts[1]
-
-  @minor.setter
-  def minor(self, value):
-    self.parts[1] = int(value)
-
-  @property
-  def maintenance(self):
-    return self.parts[2]
-
-  @maintenance.setter
-  def maintenance(self, value):
-    self.parts[2] = value
-
-  def __str__(self):
-    return '.'.join([str(p) for p in self.parts])
-
-  def __repr__(self):
-    return '<Version %s>' % str(self)
-
-  def __cmp__(self, other):
-    for i in xrange(0, 3):
-      x = self.parts[i] - other.parts[i]
-      if x != 0:
-        return -1 if x < 0 else 1
-
-    return 0
+from ostack_validator.common import Inspection, MarkedError, Mark, Version, find, index
 
 class SchemaUpdateRecord(object):
   # checkpoint's data is version number
@@ -118,6 +70,9 @@ class ConfigSchemaRegistry:
     fullname = '%s/%s' % (project, configname)
     version = Version(version)
 
+    if not fullname in self.__schemas:
+      return None
+
     records = self.__schemas[fullname]
     i = len(records)-1
     # Find latest checkpoint prior given version
@@ -218,7 +173,9 @@ def validate_enum(s, values=[]):
     return None
   return InvalidValueError('Invalid value: valid values are: %s' % ', '.join(values))
 
+@type_validator('host')
 @type_validator('string')
+@type_validator('stringlist')
 def validate_string(s):
   return None
 

ostack_validator/schemas/nova/__init__.py

@@ -0,0 +1,3 @@
+
+import ostack_validator.schemas.nova.v2013_1
+

ostack_validator/test_model_parser.py

@@ -0,0 +1,21 @@
+from ostack_validator.model_parser import ModelParser
+
+import unittest
+
+class ModelParserTests(unittest.TestCase):
+  def test_sample(self):
+    parser = ModelParser()
+
+    model = parser.parse('config')
+
+    for host in model.hosts:
+      print('Host %s' % host.name)
+
+      for component in host.components:
+        print('Component %s version %s' % (component.name, component.version))
+
+        print(component.get_config())
+
+if __name__ == '__main__':
+  unittest.main()
+