Merged OpenstackComponent classes from discovery and model submodules

This commit is contained in:
Maxim Kulkin 2013-10-01 11:47:26 +04:00
parent 576cc34fb0
commit caa8262697
8 changed files with 237 additions and 304 deletions

View File

@ -2,13 +2,14 @@ import logging
from itertools import groupby
from ostack_validator.common import Issue, MarkedIssue
from ostack_validator.discovery import OpenstackDiscovery, OpenstackComponent
from ostack_validator.model import OpenstackComponent
from ostack_validator.discovery import OpenstackDiscovery
from ostack_validator.inspection import Inspection
from ostack_validator.inspections import KeystoneAuthtokenSettingsInspection
def print_components(openstack):
for host in openstack.hosts:
print('Host %s (addresses = %s):' % (host.name, ', '.join(host.network_addresses)))
print('Host %s (id = %s, addresses = %s):' % (host.name, host.id, host.network_addresses))
for service in host.components:
print('Service %s version %s config %s' % (service.name, service.version, service.config_path))
service.config

View File

@ -18,6 +18,12 @@ def all_subclasses(klass):
subclasses.extend(all_subclasses(d))
return subclasses
def path_relative_to(path, base_path):
if not path.startswith('/'):
path = os.path.join(base_path, paste_config_path)
return path
class Version:
def __init__(self, major, minor=0, maintenance=0):
"Create Version object by either passing 3 integers, one string or an another Version object"

View File

@ -1,7 +1,7 @@
import re
from StringIO import StringIO
from ostack_validator.model import *
from ostack_validator.config_model import *
from ostack_validator.config_formats.common import *
class IniConfigParser:

View File

@ -0,0 +1,68 @@
from ostack_validator.common import Mark
class Element(object):
def __init__(self, start_mark, end_mark):
self.start_mark = start_mark
self.end_mark = end_mark
def __eq__(self, other):
return (self.__class__ == other.__class__) and (self.start_mark == other.start_mark) and (self.end_mark == other.end_mark)
def __ne__(self, other):
return not self == other
class ComponentConfig(Element):
def __init__(self, start_mark, end_mark, name, sections=[], errors=[]):
super(ComponentConfig, self).__init__(start_mark, end_mark)
self.name = name
self.sections = sections
for section in self.sections:
section.parent = self
self.errors = errors
class TextElement(Element):
def __init__(self, start_mark, end_mark, text):
super(TextElement, self).__init__(start_mark, end_mark)
self.text = text
class ConfigSection(Element):
def __init__(self, start_mark, end_mark, name, parameters):
super(ConfigSection, self).__init__(start_mark, end_mark)
self.name = name
self.parameters = parameters
for parameter in self.parameters:
parameter.parent = self
class ConfigSectionName(TextElement): pass
class ConfigParameter(Element):
def __init__(self, start_mark, end_mark, name, value, delimiter):
super(ConfigParameter, self).__init__(start_mark, end_mark)
self.name = name
self.name.parent = self
self.value = value
self.value.parent = self
self.delimiter = delimiter
self.delimiter.parent = self
def __eq__(self, other):
return (self.name.text == other.name.text) and (self.value.text == other.value.text)
def __ne__(self, other):
return not self == other
def __repr__(self):
return "<ConfigParameter %s=%s delimiter=%s>" % (self.name.text, self.value.text, self.delimiter.text)
class ConfigParameterName(TextElement): pass
class ConfigParameterValue(TextElement):
def __init__(self, start_mark, end_mark, text, value=None, quotechar=None):
super(ConfigParameterValue, self).__init__(start_mark, end_mark, text)
self.value = value
self.quotechar = quotechar

View File

@ -3,23 +3,13 @@ import re
import sys
import tempfile
import logging
from itertools import groupby
import spur
from ostack_validator.common import Issue, Mark, MarkedIssue, index
from ostack_validator.model import Openstack, Host
from ostack_validator.schema import ConfigSchemaRegistry, TypeValidatorRegistry
import ostack_validator.schemas
from ostack_validator.config_formats import IniConfigParser
from ostack_validator.model import Openstack, Host, OpenstackComponent, KeystoneComponent, NovaComputeComponent, GlanceApiComponent
def path_relative_to(path, base_path):
if not path.startswith('/'):
path = os.path.join(base_path, paste_config_path)
return path
class NodeClient(object):
def __init__(self, node_address, username, private_key_file):
@ -32,170 +22,6 @@ class NodeClient(object):
def open(self, path, mode='r'):
return self.shell.open(path, mode)
class Service(object): pass
class OpenstackComponent(Service):
logger = logging.getLogger('ostack_validator.model.openstack_component')
component = None
def __init__(self, config_path):
super(OpenstackComponent, self).__init__()
self.config_path = config_path
self.config_dir = os.path.dirname(config_path)
@property
def host(self):
return self.parent
@property
def openstack(self):
return self.host.openstack
@property
def config(self):
if not hasattr(self, '_config'):
schema = ConfigSchemaRegistry.get_schema(self.component, self.version)
if not schema:
self.logger.debug('No schema for component "%s" main config version %s. Skipping it' % (self.component, self.version))
self._config = None
else:
with self.host.client.open(self.config_path) as f:
config_contents = f.read()
self._config = self._parse_config_file(Mark('%s:%s' % (self.host.name, self.config_path)), config_contents, schema, self.openstack)
return self._config
@property
def version(self):
if not hasattr(self, '_version'):
result = self.host.client.run(['python', '-c', 'import pkg_resources; version = pkg_resources.get_provider(pkg_resources.Requirement.parse("%s")).version; print(version)' % self.component])
s = result.output.strip()
parts = []
for p in s.split('.'):
if not p[0].isdigit(): break
parts.append(p)
self._version = '.'.join(parts)
return self._version
def _parse_config_file(self, base_mark, config_contents, schema=None, issue_reporter=None):
if issue_reporter:
def report_issue(issue):
issue_reporter.report_issue(issue)
else:
def report_issue(issue): pass
_config = dict()
# Apply defaults
if schema:
for parameter in schema.parameters:
if not parameter.default: continue
if not parameter.section in _config:
_config[parameter.section] = {}
if parameter.name in _config[parameter.section]: continue
_config[parameter.section][parameter.name] = parameter.default
# Parse config file
config_parser = IniConfigParser()
parsed_config = config_parser.parse('', base_mark, config_contents)
for error in parsed_config.errors:
report_issue(error)
# Validate config parameters and store them
section_name_text_f = lambda s: s.name.text
sections_by_name = groupby(sorted(parsed_config.sections, key=section_name_text_f), key=section_name_text_f)
for section_name, sections in sections_by_name:
sections = list(sections)
if len(sections) > 1:
report_issue(Issue(Issue.INFO, 'Section "%s" appears multiple times' % section_name))
seen_parameters = set()
for section in sections:
unknown_section = False
if schema:
unknown_section = not schema.has_section(section.name.text)
if unknown_section:
report_issue(MarkedIssue(Issue.WARNING, 'Unknown section "%s"' % (section_name), section.start_mark))
continue
for parameter in section.parameters:
parameter_schema = None
if schema:
parameter_schema = schema.get_parameter(name=parameter.name.text, section=section.name.text)
if not (parameter_schema or unknown_section):
report_issue(MarkedIssue(Issue.WARNING, 'Unknown parameter: section "%s" name "%s"' % (section_name, parameter.name.text), parameter.start_mark))
continue
if parameter.name.text in seen_parameters:
report_issue(MarkedIssue(Issue.WARNING, 'Parameter "%s" in section "%s" redeclared' % (parameter.name.text, section_name), parameter.start_mark))
else:
seen_parameters.add(parameter.name.text)
if parameter_schema:
type_validator = TypeValidatorRegistry.get_validator(parameter_schema.type)
type_validation_result = type_validator.validate(parameter.value.text)
if isinstance(type_validation_result, Issue):
type_validation_result.mark = parameter.value.start_mark.merge(type_validation_result.mark)
report_issue(type_validation_result)
else:
value = type_validation_result
if not section_name in _config: _config[section_name] = {}
_config[section_name][parameter.name.text] = value
# if value == parameter_schema.default:
# report_issue(MarkedIssue(Issue.INFO, 'Explicit value equals default: section "%s" parameter "%s"' % (section_name, parameter.name.text), parameter.start_mark))
else:
if not section_name in _config: _config[section_name] = {}
_config[section_name][parameter.name.text] = parameter.value.text
return _config
class KeystoneComponent(OpenstackComponent):
component = 'keystone'
name = 'keystone'
class GlanceApiComponent(OpenstackComponent):
component = 'glance'
name = 'glance-api'
class NovaComputeComponent(OpenstackComponent):
component = 'nova'
name = 'nova-compute'
@property
def paste_config(self):
if not hasattr(self, '_paste_config'):
paste_config_path = path_relative_to(self.config['DEFAULT']['api_paste_config'], self.config_dir)
with self.host.client.open(paste_config_path) as f:
paste_config_contents = f.read()
self._paste_config = self._parse_config_file(
Mark('%s:%s' % (self.host.name, paste_config_path)),
paste_config_contents,
issue_reporter=self.openstack
)
return self._paste_config
python_re = re.compile('(/?([^/]*/)*)python[0-9.]*')
class OpenstackDiscovery(object):

View File

@ -1,7 +1,12 @@
import os.path
import re
import logging
from itertools import groupby
from ostack_validator.common import Mark
from ostack_validator.common import Mark, Issue, MarkedIssue, path_relative_to
from ostack_validator.schema import ConfigSchemaRegistry, TypeValidatorRegistry
import ostack_validator.schemas
from ostack_validator.config_formats import IniConfigParser
class IssueReporter(object):
def __init__(self):
@ -44,6 +49,16 @@ class Host(object):
def openstack(self):
return self.parent
@property
def id(self):
ether_re = re.compile('link/ether (([0-9a-f]{2}:){5}([0-9a-f]{2})) ')
result = self.client.run(['bash', '-c', 'ip link | grep "link/ether "'])
macs = []
for match in ether_re.finditer(result.output):
macs.append(match.group(1).replace(':', ''))
return ''.join(macs)
@property
def network_addresses(self):
ipaddr_re = re.compile('inet (\d+\.\d+\.\d+\.\d+)/\d+')
@ -62,13 +77,16 @@ class Host(object):
'parent': self.parent
}
class OpenstackComponent(object):
def __init__(self, name, version, base_path=None):
class Service(object): pass
class OpenstackComponent(Service):
logger = logging.getLogger('ostack_validator.model.openstack_component')
component = None
def __init__(self, config_path):
super(OpenstackComponent, self).__init__()
self.name = name
self.version = version
self.base_path = base_path or '/etc/%s' % self.name
self.configs = {}
self.config_path = config_path
self.config_dir = os.path.dirname(config_path)
@property
def host(self):
@ -76,85 +94,150 @@ class OpenstackComponent(object):
@property
def openstack(self):
return self.host.parent
return self.host.openstack
def get_config(self, config_name=None):
if config_name is None:
config_name = '%s.conf' % self.name
if not config_name in self.configs:
resource = self.openstack.resource_locator.find_resource('file', name=os.path.join(self.base_path, config_name), host=self.host.name)
if resource:
config = self.openstack.config_parser.parse(config_name, Mark(resource.name), resource.get_contents())
self.configs[config_name] = config
@property
def config(self):
if not hasattr(self, '_config'):
schema = ConfigSchemaRegistry.get_schema(self.component, self.version)
if not schema:
self.logger.debug('No schema for component "%s" main config version %s. Skipping it' % (self.component, self.version))
self._config = None
else:
self.configs[config_name] = None
with self.host.client.open(self.config_path) as f:
config_contents = f.read()
return self.configs[config_name]
self._config = self._parse_config_file(Mark('%s:%s' % (self.host.name, self.config_path)), config_contents, schema, self.openstack)
class Element(object):
def __init__(self, start_mark, end_mark):
self.start_mark = start_mark
self.end_mark = end_mark
return self._config
def __eq__(self, other):
return (self.__class__ == other.__class__) and (self.start_mark == other.start_mark) and (self.end_mark == other.end_mark)
def __ne__(self, other):
return not self == other
@property
def version(self):
if not hasattr(self, '_version'):
result = self.host.client.run(['python', '-c', 'import pkg_resources; version = pkg_resources.get_provider(pkg_resources.Requirement.parse("%s")).version; print(version)' % self.component])
s = result.output.strip()
parts = []
for p in s.split('.'):
if not p[0].isdigit(): break
class ComponentConfig(Element):
def __init__(self, start_mark, end_mark, name, sections=[], errors=[]):
super(ComponentConfig, self).__init__(start_mark, end_mark)
self.name = name
self.sections = sections
for section in self.sections:
section.parent = self
parts.append(p)
self.errors = errors
self._version = '.'.join(parts)
class TextElement(Element):
def __init__(self, start_mark, end_mark, text):
super(TextElement, self).__init__(start_mark, end_mark)
self.text = text
class ConfigSection(Element):
def __init__(self, start_mark, end_mark, name, parameters):
super(ConfigSection, self).__init__(start_mark, end_mark)
self.name = name
self.parameters = parameters
for parameter in self.parameters:
parameter.parent = self
class ConfigSectionName(TextElement): pass
class ConfigParameter(Element):
def __init__(self, start_mark, end_mark, name, value, delimiter):
super(ConfigParameter, self).__init__(start_mark, end_mark)
self.name = name
self.name.parent = self
self.value = value
self.value.parent = self
self.delimiter = delimiter
self.delimiter.parent = self
def __eq__(self, other):
return (self.name.text == other.name.text) and (self.value.text == other.value.text)
def __ne__(self, other):
return not self == other
return self._version
def __repr__(self):
return "<ConfigParameter %s=%s delimiter=%s>" % (self.name.text, self.value.text, self.delimiter.text)
def _parse_config_file(self, base_mark, config_contents, schema=None, issue_reporter=None):
if issue_reporter:
def report_issue(issue):
issue_reporter.report_issue(issue)
else:
def report_issue(issue): pass
_config = dict()
# Apply defaults
if schema:
for parameter in schema.parameters:
if not parameter.default: continue
if not parameter.section in _config:
_config[parameter.section] = {}
if parameter.name in _config[parameter.section]: continue
_config[parameter.section][parameter.name] = parameter.default
# Parse config file
config_parser = IniConfigParser()
parsed_config = config_parser.parse('', base_mark, config_contents)
for error in parsed_config.errors:
report_issue(error)
# Validate config parameters and store them
section_name_text_f = lambda s: s.name.text
sections_by_name = groupby(sorted(parsed_config.sections, key=section_name_text_f), key=section_name_text_f)
for section_name, sections in sections_by_name:
sections = list(sections)
if len(sections) > 1:
report_issue(Issue(Issue.INFO, 'Section "%s" appears multiple times' % section_name))
seen_parameters = set()
for section in sections:
unknown_section = False
if schema:
unknown_section = not schema.has_section(section.name.text)
if unknown_section:
report_issue(MarkedIssue(Issue.WARNING, 'Unknown section "%s"' % (section_name), section.start_mark))
continue
for parameter in section.parameters:
parameter_schema = None
if schema:
parameter_schema = schema.get_parameter(name=parameter.name.text, section=section.name.text)
if not (parameter_schema or unknown_section):
report_issue(MarkedIssue(Issue.WARNING, 'Unknown parameter: section "%s" name "%s"' % (section_name, parameter.name.text), parameter.start_mark))
continue
if parameter.name.text in seen_parameters:
report_issue(MarkedIssue(Issue.WARNING, 'Parameter "%s" in section "%s" redeclared' % (parameter.name.text, section_name), parameter.start_mark))
else:
seen_parameters.add(parameter.name.text)
if parameter_schema:
type_validator = TypeValidatorRegistry.get_validator(parameter_schema.type)
type_validation_result = type_validator.validate(parameter.value.text)
if isinstance(type_validation_result, Issue):
type_validation_result.mark = parameter.value.start_mark.merge(type_validation_result.mark)
report_issue(type_validation_result)
else:
value = type_validation_result
if not section_name in _config: _config[section_name] = {}
_config[section_name][parameter.name.text] = value
# if value == parameter_schema.default:
# report_issue(MarkedIssue(Issue.INFO, 'Explicit value equals default: section "%s" parameter "%s"' % (section_name, parameter.name.text), parameter.start_mark))
else:
if not section_name in _config: _config[section_name] = {}
_config[section_name][parameter.name.text] = parameter.value.text
return _config
class ConfigParameterName(TextElement): pass
class KeystoneComponent(OpenstackComponent):
component = 'keystone'
name = 'keystone'
class GlanceApiComponent(OpenstackComponent):
component = 'glance'
name = 'glance-api'
class NovaComputeComponent(OpenstackComponent):
component = 'nova'
name = 'nova-compute'
@property
def paste_config(self):
if not hasattr(self, '_paste_config'):
paste_config_path = path_relative_to(self.config['DEFAULT']['api_paste_config'], self.config_dir)
with self.host.client.open(paste_config_path) as f:
paste_config_contents = f.read()
self._paste_config = self._parse_config_file(
Mark('%s:%s' % (self.host.name, paste_config_path)),
paste_config_contents,
issue_reporter=self.openstack
)
return self._paste_config
class ConfigParameterValue(TextElement):
def __init__(self, start_mark, end_mark, text, value=None, quotechar=None):
super(ConfigParameterValue, self).__init__(start_mark, end_mark, text)
self.value = value
self.quotechar = quotechar

View File

@ -1,28 +0,0 @@
import logging
from ostack_validator.common import Version
from ostack_validator.model import *
from ostack_validator.resource import Resource, ConfigSnapshotResourceLocator
from ostack_validator.config_formats import IniConfigParser
OPENSTACK_COMPONENTS = ['nova', 'keystone', 'glance', 'cinder', 'horizon', 'quantum', 'swift']
class ModelParser(object):
logger = logging.getLogger('ostack_validator.ModelParser')
def parse(self, path):
resource_locator = ConfigSnapshotResourceLocator(path)
hosts = []
for host in resource_locator.find_resource(Resource.HOST):
components = []
for service in host.find_resource(Resource.SERVICE):
if not service.name in OPENSTACK_COMPONENTS:
continue
components.append(OpenstackComponent(service.name, service.version))
hosts.append(Host(host.name, {}, components))
return Openstack(hosts, resource_locator, IniConfigParser())

View File

@ -1,23 +0,0 @@
from ostack_validator.model_parser import ModelParser
import unittest
class ModelParserTests(unittest.TestCase):
def test_sample(self):
parser = ModelParser()
model = parser.parse('config_samples/config')
for host in model.hosts:
self.assertTrue(host.name in ['host1', 'host2'])
for component in host.components:
self.assertTrue(component.name in ['cinder', 'glance', 'horizon', 'keystone', 'nova', 'quantum', 'swift'])
self.assertTrue(len(host.components) > 0)
self.assertEqual(2, len(model.hosts))
if __name__ == '__main__':
unittest.main()