stackforge/rubick
Code Issues Proposed changes

Implemented basic config schema validation and issue reporting

Browse Source
This commit is contained in:
Maxim Kulkin 2013-09-17 14:22:26 +04:00
parent 76066c0128
commit 1cfef3b9a6
13 changed files with 168 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
ostack_validator/common.py
View File

@ -16,6 +16,9 @@ class Version:
"Create Version object by either passing 3 integers, one string or an another Version object"
if isinstance(major, str):
self.parts = [int(x) for x in major.split('.', 3)]
while len(self.parts) < 3:
self.parts.append(0)
elif isinstance(major, Version):
self.parts = major.parts
else:
@ -61,7 +64,7 @@ class Version:
class Mark(object):
def __init__(self, source, line, column):
def __init__(self, source, line=0, column=0):
self.source = source
self.line = line
self.column = column
@ -78,26 +81,41 @@ class Mark(object):
def __repr__(self):
return '%s line %d column %d' % (self.source, self.line, self.column)
class Error(object):
class Error:
def __init__(self, message):
self.message = message
def __repr__(self):
return '<%s "%s">' % (str(self.__class__).split('.')[-1][:-2], self.message)
def __str__(self):
return self.message
ERROR = 'ERROR'
WARNING = 'WARNING'
INFO = 'INFO'
class Issue(object):
def __init__(self, type, message):
self.type = type
self.message = message
def __repr__(self):
return '<%s type=%s message=%s>' % (str(self.__class__).split('.')[-1][:-2], self.type, self.message)
def __str__(self):
return 'Error: %s' % self.message
class MarkedError(Error):
def __init__(self, message, mark):
super(MarkedError, self).__init__(message)
class MarkedIssue(Issue):
def __init__(self, type, message, mark):
super(MarkedIssue, self).__init__(type, message)
self.mark = mark
def __repr__(self):
return '<%s "%s" at %s>' % (str(self.__class__).split('.')[-1][:-2], self.message, self.mark)
return '<%s type=%s message=%s mark=%s>' % (str(self.__class__).split('.')[-1][:-2], self.type, self.message, self.mark)
def __str__(self):
return self.message + (" (source '%s' line %d column %d)" % (self.mark.source, self.mark.line, self.mark.column))
return super(MarkedIssue, self).__str__() + (' (source "%s" line %d column %d)' % (self.mark.source, self.mark.line, self.mark.column))
class Inspection(object):

11
ostack_validator/config_formats/common.py
View File

@ -1,9 +1,6 @@
from ostack_validator.common import Mark, MarkedError
from ostack_validator.common import Mark, MarkedIssue, ERROR
class ParseError(MarkedError): pass
class ParseResult:
def __init__(self, success, value):
self.success = success
self.value = value
class ParseError(MarkedIssue):
def __init__(self, message, mark):
super(ParseError, self).__init__(ERROR, message, mark)

2
ostack_validator/config_formats/ini.py
View File

@ -113,7 +113,7 @@ class IniConfigParser:
sections.append(section)
parameters = []
config = ComponentConfig(name, sections, errors)
config = ComponentConfig(name, Mark(name), sections, errors)
return config

38
ostack_validator/inspection.py
View File

@ -1,12 +1,12 @@
from common import Error, MarkedError, Mark
from model import *
import logging
import unittest
from ostack_validator.common import Inspection
from ostack_validator.schema import ConfigSchemaRegistry
from ostack_validator.common import Error, Mark, Issue, Inspection
from ostack_validator.schema import ConfigSchemaRegistry, TypeValidatorRegistry
import ostack_validator.schemas
class MainConfigValidationInspection(Inspection):
logger = logging.getLogger('ostack_validator.inspections.main_config_validation')
def inspect(self, openstack):
results = []
for host in openstack.hosts:
@ -14,21 +14,29 @@ class MainConfigValidationInspection(Inspection):
main_config = component.get_config()
if not main_config:
schema.logger.debug('No main config for component "%s"' % (component.name))
results.append(Error('Missing main configuration file for component "%s" at host "%s"' % (component.name, host.name)))
continue
schema = ConfigSchemaRegistry.get_schema(component.name, component.version, main_config.name)
if not schema: continue
if not schema:
self.logger.debug('No schema for component "%s" main config version %s. Skipping it' % (component.name, component.version))
continue
for parameter in main_config.parameters:
parameter_schema = schema.get_parameter(name=parameter.name, section=parameter.section)
# TBD: should we report unknown config parameters?
if not parameter_schema: continue
for section in main_config.sections:
for parameter in section.parameters:
parameter_schema = schema.get_parameter(name=parameter.name.text, section=section.name.text)
# TBD: should we report unknown config parameters?
if not parameter_schema:
self.logger.debug('No schema for parameter "%s" in section "%s". Skipping it' % (parameter.name.text, section.name.text))
continue
type_descriptor = TypeDescriptorRepository.get_type(parameter_schema.type)
type_validation_result = type_descriptor.validate(parameter.value)
if type_validation_result:
results.append(type_validation_result)
type_validator = TypeValidatorRegistry.get_validator(parameter_schema.type)
type_validation_result = type_validator.validate(parameter.value.text)
if isinstance(type_validation_result, Issue):
self.logger.debug('Got issue for parameter "%s" with value "%s"' % (parameter.name.text, parameter.value.text))
type_validation_result.mark = main_config.mark.merge(parameter.value.start_mark.merge(type_validation_result.mark))
results.append(type_validation_result)
return results

31
ostack_validator/main.py
View File

@ -1,25 +1,40 @@
import sys
import logging
import argparse
from ostack_validator.model_parser import ModelParser
from ostack_validator.inspection import MainConfigValidationInspection
def main(args):
if len(args) < 1:
print("Usage: validator <config-snapshot-path>")
sys.exit(1)
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--debug', help='set debug log level', action='store_true')
parser.add_argument('path', help='Path to config snapshot')
args = parser.parse_args(args)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARN)
model_parser = ModelParser()
model = model_parser.parse(args[0])
print('Analyzing configs in "%s"' % args.path)
model = model_parser.parse(args.path)
inspections = [MainConfigValidationInspection()]
results = []
issues = []
for inspection in inspections:
results.extend(inspection.inspect(model))
issues.extend(inspection.inspect(model))
for result in results:
print(result)
if len(issues) == 0:
print('No issues found')
else:
for issue in issues:
print(issue)
if __name__ == '__main__':
main(sys.argv[1:])

5
ostack_validator/model.py
View File

@ -1,3 +1,4 @@
from ostack_validator.common import Mark
class Openstack(object):
def __init__(self, hosts, resource_locator, config_parser):
@ -40,6 +41,7 @@ class OpenstackComponent(object):
resource = self.openstack.resource_locator.find_resource(self.host.name, self.name, config_name)
if resource:
config = self.openstack.config_parser.parse(config_name, resource.get_contents())
config.mark = Mark(resource.name)
self.configs[config_name] = config
else:
self.configs[config_name] = None
@ -47,9 +49,10 @@ class OpenstackComponent(object):
return self.configs[config_name]
class ComponentConfig(object):
def __init__(self, name, sections=[], errors=[]):
def __init__(self, name, mark, sections=[], errors=[]):
super(ComponentConfig, self).__init__()
self.name = name
self.mark = mark
self.sections = sections
for section in self.sections:
section.parent = self

6
ostack_validator/resource.py
View File

@ -1,6 +1,8 @@
import glob
import os.path
from ostack_validator.common import Error
class Resource(object):
def __init__(self, name):
super(Resource, self).__init__()
@ -52,5 +54,7 @@ class ConfigSnapshotResourceLocator(object):
if not os.path.exists(path):
return None
return FileResource(name, path)
fullname = '%s/%s/%s' % (host, component, name)
return FileResource(fullname, path)

55
ostack_validator/schema.py
View File

@ -1,4 +1,6 @@
from ostack_validator.common import Inspection, MarkedError, Mark, Version, find, index
import sys
from ostack_validator.common import Inspection, MarkedIssue, ERROR, Mark, Version, find, index
class SchemaUpdateRecord(object):
# checkpoint's data is version number
@ -11,9 +13,13 @@ class SchemaUpdateRecord(object):
self.operation = operation
self.data = data
def __repr__(self):
return '<SchemaUpdateRecord %s %s %s' % (self.version, self.operation, self.data)
class SchemaBuilder(object):
def __init__(self, data):
def __init__(self, name, data):
super(SchemaBuilder, self).__init__()
self.name = name
self.data = data
self.current_version = None
@ -21,6 +27,10 @@ class SchemaBuilder(object):
self.adds = []
self.removals = []
def __del__(self):
if len(self.adds) > 0 or len(self.removals) > 0:
sys.stderr.write("WARNING: Uncommitted config schema \"%s\" version %s\n" % (self.name, self.current_version))
def version(self, version, checkpoint=False):
version = Version(version)
@ -36,16 +46,22 @@ class SchemaBuilder(object):
self.current_section = name
def param(self, *args, **kwargs):
self._ensure_version()
if not 'section' in kwargs and self.current_section:
kwargs['section'] = self.current_section
self.adds.append(ConfigParameterSchema(*args, **kwargs))
def remove_param(self, name):
self._ensure_version()
self.removals.append(name)
def commit(self):
"Finalize schema building"
self._ensure_version()
if len(self.removals) > 0:
self.data.append(SchemaUpdateRecord(self.current_version, 'remove', self.removals))
self.removals = []
@ -53,6 +69,10 @@ class SchemaBuilder(object):
self.data.append(SchemaUpdateRecord(self.current_version, 'add', self.adds))
self.adds = []
def _ensure_version(self):
if not self.current_version:
raise Error, 'Schema version is not specified. Please call version() method first'
class ConfigSchemaRegistry:
__schemas = {}
@classmethod
@ -61,7 +81,7 @@ class ConfigSchemaRegistry:
configname = '%s.conf' % project
fullname = '%s/%s' % (project, configname)
self.__schemas[fullname] = []
return SchemaBuilder(self.__schemas[fullname])
return SchemaBuilder(fullname, self.__schemas[fullname])
@classmethod
def get_schema(self, project, version, configname=None):
@ -114,7 +134,7 @@ class ConfigSchema:
self.format = format
self.parameters = parameters
def get_parameter(name, section=None):
def get_parameter(self, name, section=None):
# TODO: optimize this
return find(self.parameters, lambda p: p.name == name and p.section == section)
@ -145,9 +165,9 @@ class TypeValidatorRegistry:
return self.__validators[name]
class InvalidValueError(MarkedError):
class InvalidValueError(MarkedIssue):
def __init__(self, message, mark=Mark('', 1, 1)):
super(InvalidValueError, self).__init__(message, mark)
super(InvalidValueError, self).__init__(ERROR, message, mark)
class TypeValidator(object):
def __init__(self, f):
@ -167,17 +187,32 @@ def type_validator(name, **kwargs):
return fn
return wrap
@type_validator('boolean', values=['True', 'False'])
@type_validator('boolean')
def validate_boolean(s):
s = s.lower()
if s == 'true':
return True
elif s == 'false':
return False
else:
return InvalidValueError('Invalid value: value should be "true" or "false"')
def validate_enum(s, values=[]):
if s in values:
return None
return InvalidValueError('Invalid value: valid values are: %s' % ', '.join(values))
if len(values) == 0:
message = 'there should be no value'
elif len(values) == 1:
message = 'the only valid value is %s' % values[0]
else:
message = 'valid values are %s and %s' % (', '.join(values[:-1]), values[-1])
return InvalidValueError('Invalid value: %s' % message)
@type_validator('host')
@type_validator('string')
@type_validator('stringlist')
def validate_string(s):
return None
return s
@type_validator('integer')
@type_validator('port', min=1, max=65535)
@ -196,6 +231,6 @@ def validate_integer(s, min=None, max=None):
if max and v > max:
return InvalidValueError('Invalid value: should be less than or equal to %d' % max, Mark('', 1, leading_whitespace_len))
return None
return v

2
ostack_validator/schemas/__init__.py
View File

@ -0,0 +1,2 @@
import ostack_validator.schemas.nova

3
ostack_validator/schemas/nova/v2013_1.py
View File

@ -1376,9 +1376,11 @@ nova.param('use_ipv6', type='boolean')
# Print more verbose output (set logging level to INFO instead
# of default WARNING level). (boolean value)
#verbose=false
nova.param('verbose', type='boolean', default=False, description='Print more verbose output (set logging level to INFO instead of default WARNING level)')
# Log output to standard error (boolean value)
#use_stderr=true
nova.param('use_stderr', type='boolean', default=True, description='Log output to standard error')
# format string to use for log messages with context (string
# value)
@ -3387,4 +3389,5 @@ nova.section('spice')
# keymap for spice (string value)
#keymap=en-us
nova.commit()

2
ostack_validator/test_model_parser.py
View File

@ -6,7 +6,7 @@ class ModelParserTests(unittest.TestCase):
def test_sample(self):
parser = ModelParser()
model = parser.parse('config')
model = parser.parse('config_samples/config')
for host in model.hosts:
print('Host %s' % host.name)

28
ostack_validator/test_type_validators.py
View File

@ -1,4 +1,4 @@
from ostack_validator.common import Error, MarkedError
from ostack_validator.common import Issue, MarkedIssue
from ostack_validator.schema import TypeValidatorRegistry
import unittest
@ -9,10 +9,10 @@ class TypeValidatorTestHelper(object):
self.validator = TypeValidatorRegistry.get_validator(self.type_name)
def assertValid(self, value):
self.assertIsNone(self.validator.validate(value))
self.assertNotIsInstance(self.validator.validate(value), Issue)
def assertInvalid(self, value):
self.assertIsInstance(self.validator.validate(value), Error)
self.assertIsInstance(self.validator.validate(value), Issue)
class StringTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'string'
@ -23,14 +23,20 @@ class StringTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
def test_validation_always_passes(self):
self.assertValid('foo bar')
def test_should_return_same_string_if_valid(self):
s = 'foo bar'
self.assertEqual(s, self.validator.validate(s))
class BooleanTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'boolean'
def test_True(self):
self.assertValid('True')
v = self.validator.validate('True')
self.assertEqual(True, v)
def test_False(self):
self.assertValid('False')
v = self.validator.validate('False')
self.assertEqual(False, v)
def test_other_values_produce_error(self):
self.assertInvalid('foo')
@ -58,14 +64,18 @@ class IntegerTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
def test_invalid_char_error_contains_proper_column_in_mark(self):
error = self.validator.validate('12a45')
self.assertIsInstance(error, MarkedError)
self.assertIsInstance(error, MarkedIssue)
self.assertEqual(3, error.mark.column)
def test_invalid_char_error_contains_proper_column_if_leading_whitespaces(self):
error = self.validator.validate(' 12a45')
self.assertIsInstance(error, MarkedError)
self.assertIsInstance(error, MarkedIssue)
self.assertEqual(5, error.mark.column)
def test_returns_integer_if_valid(self):
v = self.validator.validate('123')
self.assertEqual(123, v)
class PortTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'port'
@ -95,6 +105,10 @@ class PortTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
self.assertValid('456 ')
self.assertValid(' 123 ')
def test_returns_integer_if_valid(self):
v = self.validator.validate('123')
self.assertEqual(123, v)
if __name__ == '__main__':
unittest.main()

11
ostack_validator/test_version.py
View File

@ -16,6 +16,17 @@ class VersionTests(unittest.TestCase):
self.assertEqual(2, v.minor)
self.assertEqual(12, v.maintenance)
def test_creation_from_string_with_less_parts(self):
v = Version('1.2')
self.assertEqual(1, v.major)
self.assertEqual(2, v.minor)
self.assertEqual(0, v.maintenance)
v = Version('12')
self.assertEqual(12, v.major)
self.assertEqual(0, v.minor)
self.assertEqual(0, v.maintenance)
def test_creation_from_other_version(self):
v = Version('1.2.3')
v2 = Version(v)