Merged OpenstackComponent classes from discovery and model submodules

discover_test.py

@@ -2,13 +2,14 @@ import logging
 from itertools import groupby
 
 from ostack_validator.common import Issue, MarkedIssue
-from ostack_validator.discovery import OpenstackDiscovery, OpenstackComponent
+from ostack_validator.model import OpenstackComponent
+from ostack_validator.discovery import OpenstackDiscovery
 from ostack_validator.inspection import Inspection
 from ostack_validator.inspections import KeystoneAuthtokenSettingsInspection
 
 def print_components(openstack):
   for host in openstack.hosts:
-    print('Host %s (addresses = %s):' % (host.name, ', '.join(host.network_addresses)))
+    print('Host %s (id = %s, addresses = %s):' % (host.name, host.id, host.network_addresses))
     for service in host.components:
       print('Service %s version %s config %s' % (service.name, service.version, service.config_path))
       service.config

ostack_validator/common.py

@@ -18,6 +18,12 @@ def all_subclasses(klass):
     subclasses.extend(all_subclasses(d))
   return subclasses
 
+def path_relative_to(path, base_path):
+  if not path.startswith('/'):
+    path = os.path.join(base_path, paste_config_path)
+
+  return path
+
 class Version:
   def __init__(self, major, minor=0, maintenance=0):
     "Create Version object by either passing 3 integers, one string or an another Version object"

ostack_validator/config_formats/ini.py

@@ -1,7 +1,7 @@
 import re
 from StringIO import StringIO
 
-from ostack_validator.model import *
+from ostack_validator.config_model import *
 from ostack_validator.config_formats.common import *
 
 class IniConfigParser:

ostack_validator/config_model.py

@@ -0,0 +1,68 @@
+from ostack_validator.common import Mark
+
+class Element(object):
+  def __init__(self, start_mark, end_mark):
+    self.start_mark = start_mark
+    self.end_mark = end_mark
+
+  def __eq__(self, other):
+    return (self.__class__ == other.__class__) and (self.start_mark == other.start_mark) and (self.end_mark == other.end_mark)
+
+  def __ne__(self, other):
+    return not self == other
+
+class ComponentConfig(Element):
+  def __init__(self, start_mark, end_mark, name, sections=[], errors=[]):
+    super(ComponentConfig, self).__init__(start_mark, end_mark)
+    self.name = name
+    self.sections = sections
+    for section in self.sections:
+      section.parent = self
+
+    self.errors = errors
+
+class TextElement(Element):
+  def __init__(self, start_mark, end_mark, text):
+    super(TextElement, self).__init__(start_mark, end_mark)
+    self.text = text
+
+class ConfigSection(Element):
+  def __init__(self, start_mark, end_mark, name, parameters):
+    super(ConfigSection, self).__init__(start_mark, end_mark)
+    self.name = name
+    self.parameters = parameters
+    for parameter in self.parameters:
+      parameter.parent = self
+
+class ConfigSectionName(TextElement): pass
+
+class ConfigParameter(Element):
+  def __init__(self, start_mark, end_mark, name, value, delimiter):
+    super(ConfigParameter, self).__init__(start_mark, end_mark)
+    self.name = name
+    self.name.parent = self
+
+    self.value = value
+    self.value.parent = self
+
+    self.delimiter = delimiter
+    self.delimiter.parent = self
+
+  def __eq__(self, other):
+    return (self.name.text == other.name.text) and (self.value.text == other.value.text)
+
+  def __ne__(self, other):
+    return not self == other
+    
+  def __repr__(self):
+    return "<ConfigParameter %s=%s delimiter=%s>" % (self.name.text, self.value.text, self.delimiter.text)
+
+
+class ConfigParameterName(TextElement): pass
+
+class ConfigParameterValue(TextElement):
+  def __init__(self, start_mark, end_mark, text, value=None, quotechar=None):
+    super(ConfigParameterValue, self).__init__(start_mark, end_mark, text)
+    self.value = value
+    self.quotechar = quotechar
+

ostack_validator/discovery.py

@@ -3,23 +3,13 @@ import re
 import sys
 import tempfile
 import logging
-from itertools import groupby
 
 import spur
 
 from ostack_validator.common import Issue, Mark, MarkedIssue, index
-from ostack_validator.model import Openstack, Host
+from ostack_validator.model import Openstack, Host, OpenstackComponent, KeystoneComponent, NovaComputeComponent, GlanceApiComponent
-from ostack_validator.schema import ConfigSchemaRegistry, TypeValidatorRegistry
-import ostack_validator.schemas
-from ostack_validator.config_formats import IniConfigParser
 
 
-def path_relative_to(path, base_path):
-  if not path.startswith('/'):
-    path = os.path.join(base_path, paste_config_path)
-
-  return path
-
 
 class NodeClient(object):
   def __init__(self, node_address, username, private_key_file):
@@ -32,170 +22,6 @@ class NodeClient(object):
   def open(self, path, mode='r'):
     return self.shell.open(path, mode)
 
-class Service(object): pass
-
-class OpenstackComponent(Service):
-  logger = logging.getLogger('ostack_validator.model.openstack_component')
-  component = None
-
-  def __init__(self, config_path):
-    super(OpenstackComponent, self).__init__()
-    self.config_path = config_path
-    self.config_dir = os.path.dirname(config_path)
-
-  @property
-  def host(self):
-    return self.parent
-
-  @property
-  def openstack(self):
-    return self.host.openstack
-
-  @property
-  def config(self):
-    if not hasattr(self, '_config'):
-      schema = ConfigSchemaRegistry.get_schema(self.component, self.version)
-      if not schema:
-        self.logger.debug('No schema for component "%s" main config version %s. Skipping it' % (self.component, self.version))
-        self._config = None
-      else:
-        with self.host.client.open(self.config_path) as f:
-          config_contents = f.read()
-
-        self._config = self._parse_config_file(Mark('%s:%s' % (self.host.name, self.config_path)), config_contents, schema, self.openstack)
-
-    return self._config
-
-
-  @property
-  def version(self):
-    if not hasattr(self, '_version'):
-      result = self.host.client.run(['python', '-c', 'import pkg_resources; version = pkg_resources.get_provider(pkg_resources.Requirement.parse("%s")).version; print(version)' % self.component])
-      
-      s = result.output.strip()
-      parts = []
-      for p in s.split('.'):
-        if not p[0].isdigit(): break
-
-        parts.append(p)
-
-      self._version = '.'.join(parts)
-
-    return self._version
-    
-
-  def _parse_config_file(self, base_mark, config_contents, schema=None, issue_reporter=None):
-    if issue_reporter:
-      def report_issue(issue):
-        issue_reporter.report_issue(issue)
-    else:
-      def report_issue(issue): pass
-
-    _config = dict()
-
-    # Apply defaults
-    if schema:
-      for parameter in schema.parameters:
-        if not parameter.default: continue
-
-        if not parameter.section in _config:
-          _config[parameter.section] = {}
-
-        if parameter.name in _config[parameter.section]: continue
-
-        _config[parameter.section][parameter.name] = parameter.default
-      
-    # Parse config file
-
-    config_parser = IniConfigParser()
-    parsed_config = config_parser.parse('', base_mark, config_contents)
-    for error in parsed_config.errors:
-      report_issue(error)
-
-    # Validate config parameters and store them
-    section_name_text_f = lambda s: s.name.text
-    sections_by_name = groupby(sorted(parsed_config.sections, key=section_name_text_f), key=section_name_text_f)
-
-    for section_name, sections in sections_by_name:
-      sections = list(sections)
-
-      if len(sections) > 1:
-        report_issue(Issue(Issue.INFO, 'Section "%s" appears multiple times' % section_name))
-
-      seen_parameters = set()
-
-      for section in sections:
-        unknown_section = False
-        if schema:
-          unknown_section = not schema.has_section(section.name.text)
-
-        if unknown_section:
-          report_issue(MarkedIssue(Issue.WARNING, 'Unknown section "%s"' % (section_name), section.start_mark))
-          continue
-
-        for parameter in section.parameters:
-          parameter_schema = None
-          if schema:
-            parameter_schema = schema.get_parameter(name=parameter.name.text, section=section.name.text)
-            if not (parameter_schema or unknown_section):
-              report_issue(MarkedIssue(Issue.WARNING, 'Unknown parameter: section "%s" name "%s"' % (section_name, parameter.name.text), parameter.start_mark))
-              continue
-
-          if parameter.name.text in seen_parameters:
-            report_issue(MarkedIssue(Issue.WARNING, 'Parameter "%s" in section "%s" redeclared' % (parameter.name.text, section_name), parameter.start_mark))
-          else:
-            seen_parameters.add(parameter.name.text)
-
-          if parameter_schema:
-            type_validator = TypeValidatorRegistry.get_validator(parameter_schema.type)
-            type_validation_result = type_validator.validate(parameter.value.text)
-            if isinstance(type_validation_result, Issue):
-              type_validation_result.mark = parameter.value.start_mark.merge(type_validation_result.mark)
-              report_issue(type_validation_result)
-
-            else:
-              value = type_validation_result
-
-              if not section_name in _config: _config[section_name] = {}
-              _config[section_name][parameter.name.text] = value
-
-              # if value == parameter_schema.default:
-              #   report_issue(MarkedIssue(Issue.INFO, 'Explicit value equals default: section "%s" parameter "%s"' % (section_name, parameter.name.text), parameter.start_mark))
-          else:
-            if not section_name in _config: _config[section_name] = {}
-            _config[section_name][parameter.name.text] = parameter.value.text
-
-    return _config
-
-
-class KeystoneComponent(OpenstackComponent):
-  component = 'keystone'
-  name = 'keystone'
-
-class GlanceApiComponent(OpenstackComponent):
-  component = 'glance'
-  name = 'glance-api'
-
-class NovaComputeComponent(OpenstackComponent):
-  component = 'nova'
-  name = 'nova-compute'
-
-  @property
-  def paste_config(self):
-    if not hasattr(self, '_paste_config'): 
-      paste_config_path = path_relative_to(self.config['DEFAULT']['api_paste_config'], self.config_dir)
-      with self.host.client.open(paste_config_path) as f:
-        paste_config_contents = f.read()
-
-      self._paste_config = self._parse_config_file(
-        Mark('%s:%s' % (self.host.name, paste_config_path)),
-        paste_config_contents,
-        issue_reporter=self.openstack
-      )
-
-    return self._paste_config
-
-
 python_re = re.compile('(/?([^/]*/)*)python[0-9.]*')
 
 class OpenstackDiscovery(object):

ostack_validator/model.py

@@ -1,7 +1,12 @@
 import os.path
 import re
+import logging
+from itertools import groupby
 
-from ostack_validator.common import Mark
+from ostack_validator.common import Mark, Issue, MarkedIssue, path_relative_to
+from ostack_validator.schema import ConfigSchemaRegistry, TypeValidatorRegistry
+import ostack_validator.schemas
+from ostack_validator.config_formats import IniConfigParser
 
 class IssueReporter(object):
   def __init__(self):
@@ -44,6 +49,16 @@ class Host(object):
   def openstack(self):
     return self.parent
 
+  @property
+  def id(self):
+    ether_re = re.compile('link/ether (([0-9a-f]{2}:){5}([0-9a-f]{2})) ')
+    result = self.client.run(['bash', '-c', 'ip link | grep "link/ether "'])
+    macs = []
+    for match in ether_re.finditer(result.output):
+      macs.append(match.group(1).replace(':', ''))
+    return ''.join(macs)
+    
+
   @property
   def network_addresses(self):
     ipaddr_re = re.compile('inet (\d+\.\d+\.\d+\.\d+)/\d+')
@@ -62,13 +77,16 @@ class Host(object):
       'parent': self.parent
     }
 
-class OpenstackComponent(object):
+class Service(object): pass
-  def __init__(self, name, version, base_path=None):
+
+class OpenstackComponent(Service):
+  logger = logging.getLogger('ostack_validator.model.openstack_component')
+  component = None
+
+  def __init__(self, config_path):
     super(OpenstackComponent, self).__init__()
-    self.name = name
+    self.config_path = config_path
-    self.version = version
+    self.config_dir = os.path.dirname(config_path)
-    self.base_path = base_path or '/etc/%s' % self.name
-    self.configs = {}
 
   @property
   def host(self):
@@ -76,85 +94,150 @@ class OpenstackComponent(object):
 
   @property
   def openstack(self):
-    return self.host.parent
+    return self.host.openstack
 
-  def get_config(self, config_name=None):
+  @property
-    if config_name is None:
+  def config(self):
-      config_name = '%s.conf' % self.name
+    if not hasattr(self, '_config'):
-
+      schema = ConfigSchemaRegistry.get_schema(self.component, self.version)
-    if not config_name in self.configs:
+      if not schema:
-      resource = self.openstack.resource_locator.find_resource('file', name=os.path.join(self.base_path, config_name), host=self.host.name)
+        self.logger.debug('No schema for component "%s" main config version %s. Skipping it' % (self.component, self.version))
-      if resource:
+        self._config = None
-        config = self.openstack.config_parser.parse(config_name, Mark(resource.name), resource.get_contents())
-        self.configs[config_name] = config
       else:
-        self.configs[config_name] = None
+        with self.host.client.open(self.config_path) as f:
+          config_contents = f.read()
 
-    return self.configs[config_name]
+        self._config = self._parse_config_file(Mark('%s:%s' % (self.host.name, self.config_path)), config_contents, schema, self.openstack)
 
-class Element(object):
+    return self._config
-  def __init__(self, start_mark, end_mark):
-    self.start_mark = start_mark
-    self.end_mark = end_mark
 
-  def __eq__(self, other):
-    return (self.__class__ == other.__class__) and (self.start_mark == other.start_mark) and (self.end_mark == other.end_mark)
 
-  def __ne__(self, other):
+  @property
-    return not self == other
+  def version(self):
+    if not hasattr(self, '_version'):
+      result = self.host.client.run(['python', '-c', 'import pkg_resources; version = pkg_resources.get_provider(pkg_resources.Requirement.parse("%s")).version; print(version)' % self.component])
+      
+      s = result.output.strip()
+      parts = []
+      for p in s.split('.'):
+        if not p[0].isdigit(): break
 
-class ComponentConfig(Element):
+        parts.append(p)
-  def __init__(self, start_mark, end_mark, name, sections=[], errors=[]):
-    super(ComponentConfig, self).__init__(start_mark, end_mark)
-    self.name = name
-    self.sections = sections
-    for section in self.sections:
-      section.parent = self
 
-    self.errors = errors
+      self._version = '.'.join(parts)
 
-class TextElement(Element):
+    return self._version
-  def __init__(self, start_mark, end_mark, text):
-    super(TextElement, self).__init__(start_mark, end_mark)
-    self.text = text
-
-class ConfigSection(Element):
-  def __init__(self, start_mark, end_mark, name, parameters):
-    super(ConfigSection, self).__init__(start_mark, end_mark)
-    self.name = name
-    self.parameters = parameters
-    for parameter in self.parameters:
-      parameter.parent = self
-
-class ConfigSectionName(TextElement): pass
-
-class ConfigParameter(Element):
-  def __init__(self, start_mark, end_mark, name, value, delimiter):
-    super(ConfigParameter, self).__init__(start_mark, end_mark)
-    self.name = name
-    self.name.parent = self
-
-    self.value = value
-    self.value.parent = self
-
-    self.delimiter = delimiter
-    self.delimiter.parent = self
-
-  def __eq__(self, other):
-    return (self.name.text == other.name.text) and (self.value.text == other.value.text)
-
-  def __ne__(self, other):
-    return not self == other
     
-  def __repr__(self):
+
-    return "<ConfigParameter %s=%s delimiter=%s>" % (self.name.text, self.value.text, self.delimiter.text)
+  def _parse_config_file(self, base_mark, config_contents, schema=None, issue_reporter=None):
+    if issue_reporter:
+      def report_issue(issue):
+        issue_reporter.report_issue(issue)
+    else:
+      def report_issue(issue): pass
+
+    _config = dict()
+
+    # Apply defaults
+    if schema:
+      for parameter in schema.parameters:
+        if not parameter.default: continue
+
+        if not parameter.section in _config:
+          _config[parameter.section] = {}
+
+        if parameter.name in _config[parameter.section]: continue
+
+        _config[parameter.section][parameter.name] = parameter.default
+      
+    # Parse config file
+
+    config_parser = IniConfigParser()
+    parsed_config = config_parser.parse('', base_mark, config_contents)
+    for error in parsed_config.errors:
+      report_issue(error)
+
+    # Validate config parameters and store them
+    section_name_text_f = lambda s: s.name.text
+    sections_by_name = groupby(sorted(parsed_config.sections, key=section_name_text_f), key=section_name_text_f)
+
+    for section_name, sections in sections_by_name:
+      sections = list(sections)
+
+      if len(sections) > 1:
+        report_issue(Issue(Issue.INFO, 'Section "%s" appears multiple times' % section_name))
+
+      seen_parameters = set()
+
+      for section in sections:
+        unknown_section = False
+        if schema:
+          unknown_section = not schema.has_section(section.name.text)
+
+        if unknown_section:
+          report_issue(MarkedIssue(Issue.WARNING, 'Unknown section "%s"' % (section_name), section.start_mark))
+          continue
+
+        for parameter in section.parameters:
+          parameter_schema = None
+          if schema:
+            parameter_schema = schema.get_parameter(name=parameter.name.text, section=section.name.text)
+            if not (parameter_schema or unknown_section):
+              report_issue(MarkedIssue(Issue.WARNING, 'Unknown parameter: section "%s" name "%s"' % (section_name, parameter.name.text), parameter.start_mark))
+              continue
+
+          if parameter.name.text in seen_parameters:
+            report_issue(MarkedIssue(Issue.WARNING, 'Parameter "%s" in section "%s" redeclared' % (parameter.name.text, section_name), parameter.start_mark))
+          else:
+            seen_parameters.add(parameter.name.text)
+
+          if parameter_schema:
+            type_validator = TypeValidatorRegistry.get_validator(parameter_schema.type)
+            type_validation_result = type_validator.validate(parameter.value.text)
+            if isinstance(type_validation_result, Issue):
+              type_validation_result.mark = parameter.value.start_mark.merge(type_validation_result.mark)
+              report_issue(type_validation_result)
+
+            else:
+              value = type_validation_result
+
+              if not section_name in _config: _config[section_name] = {}
+              _config[section_name][parameter.name.text] = value
+
+              # if value == parameter_schema.default:
+              #   report_issue(MarkedIssue(Issue.INFO, 'Explicit value equals default: section "%s" parameter "%s"' % (section_name, parameter.name.text), parameter.start_mark))
+          else:
+            if not section_name in _config: _config[section_name] = {}
+            _config[section_name][parameter.name.text] = parameter.value.text
+
+    return _config
 
 
-class ConfigParameterName(TextElement): pass
+class KeystoneComponent(OpenstackComponent):
+  component = 'keystone'
+  name = 'keystone'
+
+class GlanceApiComponent(OpenstackComponent):
+  component = 'glance'
+  name = 'glance-api'
+
+class NovaComputeComponent(OpenstackComponent):
+  component = 'nova'
+  name = 'nova-compute'
+
+  @property
+  def paste_config(self):
+    if not hasattr(self, '_paste_config'): 
+      paste_config_path = path_relative_to(self.config['DEFAULT']['api_paste_config'], self.config_dir)
+      with self.host.client.open(paste_config_path) as f:
+        paste_config_contents = f.read()
+
+      self._paste_config = self._parse_config_file(
+        Mark('%s:%s' % (self.host.name, paste_config_path)),
+        paste_config_contents,
+        issue_reporter=self.openstack
+      )
+
+    return self._paste_config
 
-class ConfigParameterValue(TextElement):
-  def __init__(self, start_mark, end_mark, text, value=None, quotechar=None):
-    super(ConfigParameterValue, self).__init__(start_mark, end_mark, text)
-    self.value = value
-    self.quotechar = quotechar
 

ostack_validator/model_parser.py

@@ -1,28 +0,0 @@
-import logging
-
-from ostack_validator.common import Version
-from ostack_validator.model import *
-from ostack_validator.resource import Resource, ConfigSnapshotResourceLocator
-from ostack_validator.config_formats import IniConfigParser
-
-OPENSTACK_COMPONENTS = ['nova', 'keystone', 'glance', 'cinder', 'horizon', 'quantum', 'swift']
-
-class ModelParser(object):
-  logger = logging.getLogger('ostack_validator.ModelParser')
-
-  def parse(self, path):
-    resource_locator = ConfigSnapshotResourceLocator(path)
-
-    hosts = []
-    for host in resource_locator.find_resource(Resource.HOST):
-      components = []
-      for service in host.find_resource(Resource.SERVICE):
-        if not service.name in OPENSTACK_COMPONENTS:
-          continue
-
-        components.append(OpenstackComponent(service.name, service.version))
-
-      hosts.append(Host(host.name, {}, components))
-
-    return Openstack(hosts, resource_locator, IniConfigParser())
-

ostack_validator/test_model_parser.py

@@ -1,23 +0,0 @@
-from ostack_validator.model_parser import ModelParser
-
-import unittest
-
-class ModelParserTests(unittest.TestCase):
-  def test_sample(self):
-    parser = ModelParser()
-
-    model = parser.parse('config_samples/config')
-
-    for host in model.hosts:
-      self.assertTrue(host.name in ['host1', 'host2'])
-
-      for component in host.components:
-        self.assertTrue(component.name in ['cinder', 'glance', 'horizon', 'keystone', 'nova', 'quantum', 'swift'])
-
-      self.assertTrue(len(host.components) > 0)
-
-    self.assertEqual(2, len(model.hosts))
-
-if __name__ == '__main__':
-  unittest.main()
-