diff --git a/ostack_validator/common.py b/ostack_validator/common.py index a4e9613..b16325e 100644 --- a/ostack_validator/common.py +++ b/ostack_validator/common.py @@ -16,6 +16,9 @@ class Version: "Create Version object by either passing 3 integers, one string or an another Version object" if isinstance(major, str): self.parts = [int(x) for x in major.split('.', 3)] + while len(self.parts) < 3: + self.parts.append(0) + elif isinstance(major, Version): self.parts = major.parts else: @@ -61,7 +64,7 @@ class Version: class Mark(object): - def __init__(self, source, line, column): + def __init__(self, source, line=0, column=0): self.source = source self.line = line self.column = column @@ -78,26 +81,41 @@ class Mark(object): def __repr__(self): return '%s line %d column %d' % (self.source, self.line, self.column) -class Error(object): +class Error: def __init__(self, message): self.message = message def __repr__(self): return '<%s "%s">' % (str(self.__class__).split('.')[-1][:-2], self.message) + def __str__(self): + return self.message + +ERROR = 'ERROR' +WARNING = 'WARNING' +INFO = 'INFO' + +class Issue(object): + def __init__(self, type, message): + self.type = type + self.message = message + + def __repr__(self): + return '<%s type=%s message=%s>' % (str(self.__class__).split('.')[-1][:-2], self.type, self.message) + def __str__(self): return 'Error: %s' % self.message -class MarkedError(Error): - def __init__(self, message, mark): - super(MarkedError, self).__init__(message) +class MarkedIssue(Issue): + def __init__(self, type, message, mark): + super(MarkedIssue, self).__init__(type, message) self.mark = mark def __repr__(self): - return '<%s "%s" at %s>' % (str(self.__class__).split('.')[-1][:-2], self.message, self.mark) + return '<%s type=%s message=%s mark=%s>' % (str(self.__class__).split('.')[-1][:-2], self.type, self.message, self.mark) def __str__(self): - return self.message + (" (source '%s' line %d column %d)" % (self.mark.source, self.mark.line, self.mark.column)) + return super(MarkedIssue, self).__str__() + (' (source "%s" line %d column %d)' % (self.mark.source, self.mark.line, self.mark.column)) class Inspection(object): diff --git a/ostack_validator/config_formats/common.py b/ostack_validator/config_formats/common.py index 9b5ec22..8a55ac8 100644 --- a/ostack_validator/config_formats/common.py +++ b/ostack_validator/config_formats/common.py @@ -1,9 +1,6 @@ -from ostack_validator.common import Mark, MarkedError +from ostack_validator.common import Mark, MarkedIssue, ERROR -class ParseError(MarkedError): pass - -class ParseResult: - def __init__(self, success, value): - self.success = success - self.value = value +class ParseError(MarkedIssue): + def __init__(self, message, mark): + super(ParseError, self).__init__(ERROR, message, mark) diff --git a/ostack_validator/config_formats/ini.py b/ostack_validator/config_formats/ini.py index d04b0f1..fd1013c 100644 --- a/ostack_validator/config_formats/ini.py +++ b/ostack_validator/config_formats/ini.py @@ -113,7 +113,7 @@ class IniConfigParser: sections.append(section) parameters = [] - config = ComponentConfig(name, sections, errors) + config = ComponentConfig(name, Mark(name), sections, errors) return config diff --git a/ostack_validator/inspection.py b/ostack_validator/inspection.py index 3892657..74f342f 100644 --- a/ostack_validator/inspection.py +++ b/ostack_validator/inspection.py @@ -1,12 +1,12 @@ -from common import Error, MarkedError, Mark -from model import * +import logging -import unittest - -from ostack_validator.common import Inspection -from ostack_validator.schema import ConfigSchemaRegistry +from ostack_validator.common import Error, Mark, Issue, Inspection +from ostack_validator.schema import ConfigSchemaRegistry, TypeValidatorRegistry +import ostack_validator.schemas class MainConfigValidationInspection(Inspection): + logger = logging.getLogger('ostack_validator.inspections.main_config_validation') + def inspect(self, openstack): results = [] for host in openstack.hosts: @@ -14,21 +14,29 @@ class MainConfigValidationInspection(Inspection): main_config = component.get_config() if not main_config: + schema.logger.debug('No main config for component "%s"' % (component.name)) results.append(Error('Missing main configuration file for component "%s" at host "%s"' % (component.name, host.name))) continue schema = ConfigSchemaRegistry.get_schema(component.name, component.version, main_config.name) - if not schema: continue + if not schema: + self.logger.debug('No schema for component "%s" main config version %s. Skipping it' % (component.name, component.version)) + continue - for parameter in main_config.parameters: - parameter_schema = schema.get_parameter(name=parameter.name, section=parameter.section) - # TBD: should we report unknown config parameters? - if not parameter_schema: continue + for section in main_config.sections: + for parameter in section.parameters: + parameter_schema = schema.get_parameter(name=parameter.name.text, section=section.name.text) + # TBD: should we report unknown config parameters? + if not parameter_schema: + self.logger.debug('No schema for parameter "%s" in section "%s". Skipping it' % (parameter.name.text, section.name.text)) + continue - type_descriptor = TypeDescriptorRepository.get_type(parameter_schema.type) - type_validation_result = type_descriptor.validate(parameter.value) - if type_validation_result: - results.append(type_validation_result) + type_validator = TypeValidatorRegistry.get_validator(parameter_schema.type) + type_validation_result = type_validator.validate(parameter.value.text) + if isinstance(type_validation_result, Issue): + self.logger.debug('Got issue for parameter "%s" with value "%s"' % (parameter.name.text, parameter.value.text)) + type_validation_result.mark = main_config.mark.merge(parameter.value.start_mark.merge(type_validation_result.mark)) + results.append(type_validation_result) return results diff --git a/ostack_validator/main.py b/ostack_validator/main.py index 8a082cf..35ba62c 100644 --- a/ostack_validator/main.py +++ b/ostack_validator/main.py @@ -1,25 +1,40 @@ import sys +import logging +import argparse from ostack_validator.model_parser import ModelParser from ostack_validator.inspection import MainConfigValidationInspection def main(args): - if len(args) < 1: - print("Usage: validator ") - sys.exit(1) + parser = argparse.ArgumentParser() + parser.add_argument('-d', '--debug', help='set debug log level', action='store_true') + parser.add_argument('path', help='Path to config snapshot') + + args = parser.parse_args(args) + + if args.debug: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.WARN) model_parser = ModelParser() - model = model_parser.parse(args[0]) + print('Analyzing configs in "%s"' % args.path) + + model = model_parser.parse(args.path) inspections = [MainConfigValidationInspection()] - results = [] + issues = [] for inspection in inspections: - results.extend(inspection.inspect(model)) + issues.extend(inspection.inspect(model)) - for result in results: - print(result) + if len(issues) == 0: + print('No issues found') + else: + for issue in issues: + print(issue) + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/ostack_validator/model.py b/ostack_validator/model.py index 7d54889..1072ea2 100644 --- a/ostack_validator/model.py +++ b/ostack_validator/model.py @@ -1,3 +1,4 @@ +from ostack_validator.common import Mark class Openstack(object): def __init__(self, hosts, resource_locator, config_parser): @@ -40,6 +41,7 @@ class OpenstackComponent(object): resource = self.openstack.resource_locator.find_resource(self.host.name, self.name, config_name) if resource: config = self.openstack.config_parser.parse(config_name, resource.get_contents()) + config.mark = Mark(resource.name) self.configs[config_name] = config else: self.configs[config_name] = None @@ -47,9 +49,10 @@ class OpenstackComponent(object): return self.configs[config_name] class ComponentConfig(object): - def __init__(self, name, sections=[], errors=[]): + def __init__(self, name, mark, sections=[], errors=[]): super(ComponentConfig, self).__init__() self.name = name + self.mark = mark self.sections = sections for section in self.sections: section.parent = self diff --git a/ostack_validator/resource.py b/ostack_validator/resource.py index 5d8a813..52414c2 100644 --- a/ostack_validator/resource.py +++ b/ostack_validator/resource.py @@ -1,6 +1,8 @@ import glob import os.path +from ostack_validator.common import Error + class Resource(object): def __init__(self, name): super(Resource, self).__init__() @@ -52,5 +54,7 @@ class ConfigSnapshotResourceLocator(object): if not os.path.exists(path): return None - return FileResource(name, path) + fullname = '%s/%s/%s' % (host, component, name) + + return FileResource(fullname, path) diff --git a/ostack_validator/schema.py b/ostack_validator/schema.py index a989231..b9df7af 100644 --- a/ostack_validator/schema.py +++ b/ostack_validator/schema.py @@ -1,4 +1,6 @@ -from ostack_validator.common import Inspection, MarkedError, Mark, Version, find, index +import sys + +from ostack_validator.common import Inspection, MarkedIssue, ERROR, Mark, Version, find, index class SchemaUpdateRecord(object): # checkpoint's data is version number @@ -11,9 +13,13 @@ class SchemaUpdateRecord(object): self.operation = operation self.data = data + def __repr__(self): + return ' 0 or len(self.removals) > 0: + sys.stderr.write("WARNING: Uncommitted config schema \"%s\" version %s\n" % (self.name, self.current_version)) + def version(self, version, checkpoint=False): version = Version(version) @@ -36,16 +46,22 @@ class SchemaBuilder(object): self.current_section = name def param(self, *args, **kwargs): + self._ensure_version() + if not 'section' in kwargs and self.current_section: kwargs['section'] = self.current_section self.adds.append(ConfigParameterSchema(*args, **kwargs)) def remove_param(self, name): + self._ensure_version() + self.removals.append(name) def commit(self): "Finalize schema building" + self._ensure_version() + if len(self.removals) > 0: self.data.append(SchemaUpdateRecord(self.current_version, 'remove', self.removals)) self.removals = [] @@ -53,6 +69,10 @@ class SchemaBuilder(object): self.data.append(SchemaUpdateRecord(self.current_version, 'add', self.adds)) self.adds = [] + def _ensure_version(self): + if not self.current_version: + raise Error, 'Schema version is not specified. Please call version() method first' + class ConfigSchemaRegistry: __schemas = {} @classmethod @@ -61,7 +81,7 @@ class ConfigSchemaRegistry: configname = '%s.conf' % project fullname = '%s/%s' % (project, configname) self.__schemas[fullname] = [] - return SchemaBuilder(self.__schemas[fullname]) + return SchemaBuilder(fullname, self.__schemas[fullname]) @classmethod def get_schema(self, project, version, configname=None): @@ -114,7 +134,7 @@ class ConfigSchema: self.format = format self.parameters = parameters - def get_parameter(name, section=None): + def get_parameter(self, name, section=None): # TODO: optimize this return find(self.parameters, lambda p: p.name == name and p.section == section) @@ -145,9 +165,9 @@ class TypeValidatorRegistry: return self.__validators[name] -class InvalidValueError(MarkedError): +class InvalidValueError(MarkedIssue): def __init__(self, message, mark=Mark('', 1, 1)): - super(InvalidValueError, self).__init__(message, mark) + super(InvalidValueError, self).__init__(ERROR, message, mark) class TypeValidator(object): def __init__(self, f): @@ -167,17 +187,32 @@ def type_validator(name, **kwargs): return fn return wrap -@type_validator('boolean', values=['True', 'False']) +@type_validator('boolean') +def validate_boolean(s): + s = s.lower() + if s == 'true': + return True + elif s == 'false': + return False + else: + return InvalidValueError('Invalid value: value should be "true" or "false"') + def validate_enum(s, values=[]): if s in values: return None - return InvalidValueError('Invalid value: valid values are: %s' % ', '.join(values)) + if len(values) == 0: + message = 'there should be no value' + elif len(values) == 1: + message = 'the only valid value is %s' % values[0] + else: + message = 'valid values are %s and %s' % (', '.join(values[:-1]), values[-1]) + return InvalidValueError('Invalid value: %s' % message) @type_validator('host') @type_validator('string') @type_validator('stringlist') def validate_string(s): - return None + return s @type_validator('integer') @type_validator('port', min=1, max=65535) @@ -196,6 +231,6 @@ def validate_integer(s, min=None, max=None): if max and v > max: return InvalidValueError('Invalid value: should be less than or equal to %d' % max, Mark('', 1, leading_whitespace_len)) - return None + return v diff --git a/ostack_validator/schemas/__init__.py b/ostack_validator/schemas/__init__.py index e69de29..aac3da2 100644 --- a/ostack_validator/schemas/__init__.py +++ b/ostack_validator/schemas/__init__.py @@ -0,0 +1,2 @@ +import ostack_validator.schemas.nova + diff --git a/ostack_validator/schemas/nova/v2013_1.py b/ostack_validator/schemas/nova/v2013_1.py index a5b7957..a82102a 100644 --- a/ostack_validator/schemas/nova/v2013_1.py +++ b/ostack_validator/schemas/nova/v2013_1.py @@ -1376,9 +1376,11 @@ nova.param('use_ipv6', type='boolean') # Print more verbose output (set logging level to INFO instead # of default WARNING level). (boolean value) #verbose=false +nova.param('verbose', type='boolean', default=False, description='Print more verbose output (set logging level to INFO instead of default WARNING level)') # Log output to standard error (boolean value) #use_stderr=true +nova.param('use_stderr', type='boolean', default=True, description='Log output to standard error') # format string to use for log messages with context (string # value) @@ -3387,4 +3389,5 @@ nova.section('spice') # keymap for spice (string value) #keymap=en-us +nova.commit() diff --git a/ostack_validator/test_model_parser.py b/ostack_validator/test_model_parser.py index 8e599a9..fcfb48d 100644 --- a/ostack_validator/test_model_parser.py +++ b/ostack_validator/test_model_parser.py @@ -6,7 +6,7 @@ class ModelParserTests(unittest.TestCase): def test_sample(self): parser = ModelParser() - model = parser.parse('config') + model = parser.parse('config_samples/config') for host in model.hosts: print('Host %s' % host.name) diff --git a/ostack_validator/test_type_validators.py b/ostack_validator/test_type_validators.py index 3526060..7552203 100644 --- a/ostack_validator/test_type_validators.py +++ b/ostack_validator/test_type_validators.py @@ -1,4 +1,4 @@ -from ostack_validator.common import Error, MarkedError +from ostack_validator.common import Issue, MarkedIssue from ostack_validator.schema import TypeValidatorRegistry import unittest @@ -9,10 +9,10 @@ class TypeValidatorTestHelper(object): self.validator = TypeValidatorRegistry.get_validator(self.type_name) def assertValid(self, value): - self.assertIsNone(self.validator.validate(value)) + self.assertNotIsInstance(self.validator.validate(value), Issue) def assertInvalid(self, value): - self.assertIsInstance(self.validator.validate(value), Error) + self.assertIsInstance(self.validator.validate(value), Issue) class StringTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase): type_name = 'string' @@ -23,14 +23,20 @@ class StringTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase): def test_validation_always_passes(self): self.assertValid('foo bar') + def test_should_return_same_string_if_valid(self): + s = 'foo bar' + self.assertEqual(s, self.validator.validate(s)) + class BooleanTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase): type_name = 'boolean' def test_True(self): - self.assertValid('True') + v = self.validator.validate('True') + self.assertEqual(True, v) def test_False(self): - self.assertValid('False') + v = self.validator.validate('False') + self.assertEqual(False, v) def test_other_values_produce_error(self): self.assertInvalid('foo') @@ -58,14 +64,18 @@ class IntegerTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase): def test_invalid_char_error_contains_proper_column_in_mark(self): error = self.validator.validate('12a45') - self.assertIsInstance(error, MarkedError) + self.assertIsInstance(error, MarkedIssue) self.assertEqual(3, error.mark.column) def test_invalid_char_error_contains_proper_column_if_leading_whitespaces(self): error = self.validator.validate(' 12a45') - self.assertIsInstance(error, MarkedError) + self.assertIsInstance(error, MarkedIssue) self.assertEqual(5, error.mark.column) + def test_returns_integer_if_valid(self): + v = self.validator.validate('123') + self.assertEqual(123, v) + class PortTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase): type_name = 'port' @@ -95,6 +105,10 @@ class PortTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase): self.assertValid('456 ') self.assertValid(' 123 ') + def test_returns_integer_if_valid(self): + v = self.validator.validate('123') + self.assertEqual(123, v) + if __name__ == '__main__': unittest.main() diff --git a/ostack_validator/test_version.py b/ostack_validator/test_version.py index 28f335b..4183d8a 100644 --- a/ostack_validator/test_version.py +++ b/ostack_validator/test_version.py @@ -16,6 +16,17 @@ class VersionTests(unittest.TestCase): self.assertEqual(2, v.minor) self.assertEqual(12, v.maintenance) + def test_creation_from_string_with_less_parts(self): + v = Version('1.2') + self.assertEqual(1, v.major) + self.assertEqual(2, v.minor) + self.assertEqual(0, v.maintenance) + + v = Version('12') + self.assertEqual(12, v.major) + self.assertEqual(0, v.minor) + self.assertEqual(0, v.maintenance) + def test_creation_from_other_version(self): v = Version('1.2.3') v2 = Version(v)