stackforge/rubick
Code Issues Proposed changes

PEP8 fixes

Browse Source
This commit is contained in:
Maxim Kulkin 2013-10-16 18:07:40 +04:00
parent 8ac684d600
commit a059946079
34 changed files with 10240 additions and 4887 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ostack_validator/__init__.py
View File

@ -1,6 +1,5 @@
if __name__ == '__main__':
import sys
from ostack_validator.main import main
main(sys.argv[1:])
import sys
from ostack_validator.main import main
main(sys.argv[1:])

74
ostack_validator/celery.py
View File

@ -15,51 +15,61 @@ backend_url = os.getenv('CELERY_RESULT_BACKEND', broker_url)
app = Celery('ostack_validator', broker=broker_url, backend=backend_url)
app.conf.update(
CELERY_TRACK_STARTED=True
CELERY_TRACK_STARTED=True
)
class InspectionRequest(object):
def __init__(self, nodes, username, password=None, private_key=None):
super(InspectionRequest, self).__init__()
self.nodes = nodes
self.username = username
self.password = password
self.private_key = private_key
def __init__(self, nodes, username, password=None, private_key=None):
super(InspectionRequest, self).__init__()
self.nodes = nodes
self.username = username
self.password = password
self.private_key = private_key
class InspectionResult(object):
def __init__(self, request, value):
super(InspectionResult, self).__init__()
self.request = request
self.value = value
def __init__(self, request, value):
super(InspectionResult, self).__init__()
self.request = request
self.value = value
@app.task
def ostack_inspect_task(request):
logger = logging.getLogger('ostack_validator.task.inspect')
logger = logging.getLogger('ostack_validator.task.inspect')
discovery = OpenstackDiscovery()
discovery = OpenstackDiscovery()
try:
openstack = discovery.discover(request.nodes, request.username, private_key=request.private_key)
except:
message = traceback.format_exc()
logger.error(message)
try:
openstack = discovery.discover(request.nodes, request.username,
private_key=request.private_key)
except:
message = traceback.format_exc()
logger.error(message)
return InspectionResult(request, message)
all_inspections = [KeystoneAuthtokenSettingsInspection]
for inspection in all_inspections:
try:
x = inspection()
x.inspect(openstack)
except:
message = traceback.format_exc()
logger.error(message)
openstack.report_issue(Issue(Issue.ERROR, 'Unexpected error running inspection "%s". See log for details' % inspection.name))
all_inspections = [KeystoneAuthtokenSettingsInspection]
for inspection in all_inspections:
try:
x = inspection()
x.inspect(openstack)
except:
message = traceback.format_exc()
logger.error(message)
openstack.report_issue(
Issue(
Issue.ERROR,
'Unexpected error running inspection "%s". See log for details' %
inspection.name))
return InspectionResult(request, openstack)
return InspectionResult(request, openstack)
if __name__ == '__main__':
logging.basicConfig(level=logging.WARNING)
logging.getLogger('ostack_validator').setLevel(logging.DEBUG)
app.start()
logging.basicConfig(level=logging.WARNING)
logging.getLogger('ostack_validator').setLevel(logging.DEBUG)
app.start()

238
ostack_validator/common.py
View File

@ -1,146 +1,188 @@
import copy
def find(l, predicate):
results = [x for x in l if predicate(x)]
return results[0] if len(results) > 0 else None
results = [x for x in l if predicate(x)]
return results[0] if len(results) > 0 else None
def index(l, predicate):
i = 0
while i<len(l):
if predicate(l[i]):
return i
i += 1
return -1
i = 0
while i < len(l):
if predicate(l[i]):
return i
i += 1
return -1
def all_subclasses(klass):
subclasses = klass.__subclasses__()
for d in list(subclasses):
subclasses.extend(all_subclasses(d))
return subclasses
subclasses = klass.__subclasses__()
for d in list(subclasses):
subclasses.extend(all_subclasses(d))
return subclasses
def path_relative_to(path, base_path):
if not path.startswith('/'):
path = os.path.join(base_path, paste_config_path)
if not path.startswith('/'):
path = os.path.join(base_path, paste_config_path)
return path
return path
class Version:
def __init__(self, major, minor=0, maintenance=0):
"Create Version object by either passing 3 integers, one string or an another Version object"
if isinstance(major, str):
self.parts = [int(x) for x in major.split('.', 3)]
while len(self.parts) < 3:
self.parts.append(0)
elif isinstance(major, Version):
self.parts = major.parts
else:
self.parts = [int(major), int(minor), int(maintenance)]
def __init__(self, major, minor=0, maintenance=0):
"Create Version object by either passing 3 integers, one string or an another Version object"
if isinstance(major, str):
self.parts = [int(x) for x in major.split('.', 3)]
while len(self.parts) < 3:
self.parts.append(0)
@property
def major(self):
return self.parts[0]
elif isinstance(major, Version):
self.parts = major.parts
else:
self.parts = [int(major), int(minor), int(maintenance)]
@major.setter
def major(self, value):
self.parts[0] = int(value)
@property
def major(self):
return self.parts[0]
@property
def minor(self):
return self.parts[1]
@major.setter
def major(self, value):
self.parts[0] = int(value)
@minor.setter
def minor(self, value):
self.parts[1] = int(value)
@property
def minor(self):
return self.parts[1]
@property
def maintenance(self):
return self.parts[2]
@minor.setter
def minor(self, value):
self.parts[1] = int(value)
@maintenance.setter
def maintenance(self, value):
self.parts[2] = value
@property
def maintenance(self):
return self.parts[2]
def __str__(self):
return '.'.join([str(p) for p in self.parts])
@maintenance.setter
def maintenance(self, value):
self.parts[2] = value
def __repr__(self):
return '<Version %s>' % str(self)
def __str__(self):
return '.'.join([str(p) for p in self.parts])
def __cmp__(self, other):
for i in xrange(0, 3):
x = self.parts[i] - other.parts[i]
if x != 0:
return -1 if x < 0 else 1
def __repr__(self):
return '<Version %s>' % str(self)
return 0
def __cmp__(self, other):
for i in xrange(0, 3):
x = self.parts[i] - other.parts[i]
if x != 0:
return -1 if x < 0 else 1
return 0
class Mark(object):
def __init__(self, source, line=0, column=0):
self.source = source
self.line = line
self.column = column
def __eq__(self, other):
return (self.source == source) and (self.line == other.line) and (self.column == other.column)
def __init__(self, source, line=0, column=0):
self.source = source
self.line = line
self.column = column
def __ne__(self, other):
return not self == other
def __eq__(self, other):
return (
(self.source == source) and (
self.line == other.line) and (self.column == other.column)
)
def merge(self, other):
return Mark(self.source, self.line + other.line, self.column + other.column)
def __ne__(self, other):
return not self == other
def merge(self, other):
return (
Mark(
self.source,
self.line +
other.line,
self.column +
other.column)
)
def __repr__(self):
return '%s line %d column %d' % (self.source, self.line, self.column)
def __repr__(self):
return '%s line %d column %d' % (self.source, self.line, self.column)
class Error:
def __init__(self, message):
self.message = message
def __repr__(self):
return '<%s "%s">' % (str(self.__class__).split('.')[-1][:-2], self.message)
def __init__(self, message):
self.message = message
def __repr__(self):
return (
'<%s "%s">' % (
str(self.__class__).split('.')[-1][:-2],
self.message)
)
def __str__(self):
return self.message
def __str__(self):
return self.message
class Issue(object):
FATAL = 'FATAL'
ERROR = 'ERROR'
WARNING = 'WARNING'
INFO = 'INFO'
FATAL = 'FATAL'
ERROR = 'ERROR'
WARNING = 'WARNING'
INFO = 'INFO'
def __init__(self, type, message):
self.type = type
self.message = message
def __init__(self, type, message):
self.type = type
self.message = message
def __repr__(self):
return '<%s type=%s message=%s>' % (str(self.__class__).split('.')[-1][:-2], self.type, self.message)
def __repr__(self):
return (
'<%s type=%s message=%s>' % (
str(self.__class__).split('.')[-1][:-2],
self.type,
self.message)
)
def __str__(self):
return '%s: %s' % (self.type, self.message)
def __str__(self):
return '%s: %s' % (self.type, self.message)
class MarkedIssue(Issue):
def __init__(self, type, message, mark):
super(MarkedIssue, self).__init__(type, message)
self.mark = mark
def offset_by(self, base_mark):
other = copy.copy(self)
other.mark = base_mark.merge(self.mark)
return other
def __init__(self, type, message, mark):
super(MarkedIssue, self).__init__(type, message)
self.mark = mark
def __repr__(self):
return '<%s type=%s message=%s mark=%s>' % (str(self.__class__).split('.')[-1][:-2], self.type, self.message, self.mark)
def offset_by(self, base_mark):
other = copy.copy(self)
other.mark = base_mark.merge(self.mark)
return other
def __repr__(self):
return (
'<%s type=%s message=%s mark=%s>' % (
str(self.__class__).split('.')[-1][:-2],
self.type,
self.message,
self.mark)
)
def __str__(self):
return (
super(
MarkedIssue, self).__str__() + (' (source "%s" line %d column %d)' %
(self.mark.source, self.mark.line + 1, self.mark.column + 1))
)
def __str__(self):
return super(MarkedIssue, self).__str__() + (' (source "%s" line %d column %d)' % (self.mark.source, self.mark.line+1, self.mark.column+1))
class Inspection(object):
@classmethod
def all_inspections(klass):
return [c for c in all_subclasses(klass)]
def inspect(self, openstack):
pass
@classmethod
def all_inspections(klass):
return [c for c in all_subclasses(klass)]
def inspect(self, openstack):
pass

1
ostack_validator/config_formats/__init__.py
View File

@ -1,4 +1,3 @@
from common import *
from ini import IniConfigParser

7
ostack_validator/config_formats/common.py
View File

@ -1,6 +1,7 @@
from ostack_validator.common import Mark, Issue, MarkedIssue
class ParseError(MarkedIssue):
def __init__(self, message, mark):
super(ParseError, self).__init__(Issue.ERROR, message, mark)
class ParseError(MarkedIssue):
def __init__(self, message, mark):
super(ParseError, self).__init__(Issue.ERROR, message, mark)

235
ostack_validator/config_formats/ini.py
View File

@ -4,123 +4,150 @@ from StringIO import StringIO
from ostack_validator.config_model import *
from ostack_validator.config_formats.common import *
class IniConfigParser:
key_value_re = re.compile("^(\S+?)\s*([:=])\s*('.*'|\".*\"|.*)\s*$")
key_value_re = re.compile("^(\S+?)\s*([:=])\s*('.*'|\".*\"|.*)\s*$")
def parse(self, name, base_mark, io):
if not hasattr(io, 'readlines'):
io = StringIO(io)
def parse(self, name, base_mark, io):
if not hasattr(io, 'readlines'):
io = StringIO(io)
def mark(line, column=0):
return base_mark.merge(Mark('', line, column))
errors = []
current_section_name = ConfigSectionName(mark(0), mark(0), '')
current_param_name = None
current_param_value = None
current_param_delimiter = None
sections = []
parameters = []
line_number = -1
for line in io.readlines():
line = line.rstrip()
line_number += 1
if current_param_name and (current_param_value.quotechar or (line == '' or not line[0].isspace())):
param = ConfigParameter(current_param_name.start_mark, current_param_value.end_mark, current_param_name, current_param_value, current_param_delimiter)
parameters.append(param)
def mark(line, column=0):
return base_mark.merge(Mark('', line, column))
errors = []
current_section_name = ConfigSectionName(mark(0), mark(0), '')
current_param_name = None
current_param_value = None
current_param_delimiter = None
sections = []
parameters = []
if line == '': continue
line_number = -1
for line in io.readlines():
line = line.rstrip()
if line[0] in '#;': continue
line_number += 1
if current_param_name and (current_param_value.quotechar or (line == '' or not line[0].isspace())):
param = ConfigParameter(
current_param_name.start_mark,
current_param_value.end_mark,
current_param_name,
current_param_value,
current_param_delimiter)
parameters.append(param)
current_param_name = None
current_param_value = None
current_param_delimiter = None
if line == '':
continue
if line[0] in '#;':
continue
if line[0].isspace():
if current_param_name:
current_param_value.end_mark = mark(line_number, len(line))
current_param_value.text += line.lstrip()
continue
else:
errors.append(
ParseError('Unexpected multiline value continuation', mark(line_number)))
continue
if line[0] == '[':
end_index = line.find(']')
if end_index == -1:
errors.append(
ParseError('Unclosed section', mark(line_number, len(line))))
end_index = len(line)
while line[end_index - 1].isspace():
end_index -= 1
if end_index <= 1:
errors.append(
ParseError('Missing section name', mark(line_number)))
continue
else:
i = end_index + 1
while i < len(line):
if not line[i].isspace():
errors.append(
ParseError('Extra chars after section name', mark(line_number, i)))
break
i += 1
if current_section_name.text != '' or len(parameters) > 0:
section = ConfigSection(
current_section_name.start_mark,
mark(line_number),
current_section_name,
parameters)
sections.append(section)
parameters = []
current_section_name = ConfigSectionName(
mark(line_number, 0),
mark(line_number, end_index),
line[1:end_index]
)
else:
m = self.key_value_re.match(line)
if m:
current_param_name = ConfigParameterName(
mark(line_number, m.start(1)),
mark(line_number, m.end(1)),
m.group(1)
)
current_param_delimiter = TextElement(
mark(line_number, m.start(2)),
mark(line_number, m.end(2)),
m.group(2)
)
# Unquote value
value = m.group(3)
quotechar = None
if len(value) > 0 and (value[0] == value[-1] and value[0] in "\"'"):
quotechar = value[0]
value = value[1:-1]
current_param_value = ConfigParameterValue(
mark(line_number, m.start(3)),
mark(line_number, m.end(3)),
value,
quotechar=quotechar
)
else:
errors.append(
ParseError('Syntax error in line "%s"' %
line, mark(line_number)))
if line[0].isspace():
if current_param_name:
current_param_value.end_mark = mark(line_number, len(line))
current_param_value.text += line.lstrip()
continue
else:
errors.append(ParseError('Unexpected multiline value continuation', mark(line_number)))
continue
if line[0] == '[':
end_index = line.find(']')
if end_index == -1:
errors.append(ParseError('Unclosed section', mark(line_number, len(line))))
end_index = len(line)
while line[end_index-1].isspace(): end_index -= 1
if end_index <= 1:
errors.append(ParseError('Missing section name', mark(line_number)))
continue
else:
i = end_index+1
while i < len(line):
if not line[i].isspace():
errors.append(ParseError('Extra chars after section name', mark(line_number, i)))
break
i += 1
param = ConfigParameter(
current_param_name.start_mark,
current_param_value.end_mark,
current_param_name,
current_param_value,
current_param_delimiter)
parameters.append(param)
if current_section_name.text != '' or len(parameters) > 0:
section = ConfigSection(current_section_name.start_mark, mark(line_number), current_section_name, parameters)
sections.append(section)
parameters = []
section = ConfigSection(
current_section_name.start_mark,
mark(line_number),
current_section_name,
parameters)
sections.append(section)
parameters = []
current_section_name = ConfigSectionName(
mark(line_number, 0),
mark(line_number, end_index),
line[1:end_index]
)
else:
m = self.key_value_re.match(line)
if m:
current_param_name = ConfigParameterName(
mark(line_number, m.start(1)),
mark(line_number, m.end(1)),
m.group(1)
)
current_param_delimiter = TextElement(
mark(line_number, m.start(2)),
mark(line_number, m.end(2)),
m.group(2)
)
end_mark = base_mark
if len(sections) > 0:
end_mark = base_mark.merge(sections[-1].end_mark)
# Unquote value
value = m.group(3)
quotechar = None
if len(value) > 0 and (value[0] == value[-1] and value[0] in "\"'"):
quotechar = value[0]
value = value[1:-1]
current_param_value = ConfigParameterValue(
mark(line_number, m.start(3)),
mark(line_number, m.end(3)),
value,
quotechar=quotechar
)
else:
errors.append(ParseError('Syntax error in line "%s"' % line, mark(line_number)))
if current_param_name:
param = ConfigParameter(current_param_name.start_mark, current_param_value.end_mark, current_param_name, current_param_value, current_param_delimiter)
parameters.append(param)
if current_section_name.text != '' or len(parameters) > 0:
section = ConfigSection(current_section_name.start_mark, mark(line_number), current_section_name, parameters)
sections.append(section)
parameters = []
end_mark = base_mark
if len(sections) > 0:
end_mark = base_mark.merge(sections[-1].end_mark)
config = ComponentConfig(base_mark, end_mark, name, sections, errors)
return config
config = ComponentConfig(base_mark, end_mark, name, sections, errors)
return config

285
ostack_validator/config_formats/test_ini.py
View File

@ -2,214 +2,229 @@ import unittest
from ostack_validator.config_formats.ini import *
class IniConfigParserTests(unittest.TestCase):
def setUp(self):
self.parser = IniConfigParser()
def _strip_margin(self, content):
lines = content.split("\n")
if lines[0] == '' and lines[-1].strip() == '':
lines = lines[1:-1]
first_line = lines[0]
margin_size = 0
while margin_size < len(first_line) and first_line[margin_size].isspace(): margin_size += 1
def setUp(self):
self.parser = IniConfigParser()
stripped_lines = [line[margin_size:] for line in lines]
def _strip_margin(self, content):
lines = content.split("\n")
if lines[0] == '' and lines[-1].strip() == '':
lines = lines[1:-1]
first_line = lines[0]
margin_size = 0
while margin_size < len(first_line) and first_line[margin_size].isspace():
margin_size += 1
return "\n".join(stripped_lines)
stripped_lines = [line[margin_size:] for line in lines]
def parse(self, content, margin=False):
if margin:
content = self._strip_margin(content)
return "\n".join(stripped_lines)
return self.parser.parse('test.conf', Mark(''), content)
def parse(self, content, margin=False):
if margin:
content = self._strip_margin(content)
def test_parsing(self):
config = self.parse("param1 = value1")
return self.parser.parse('test.conf', Mark(''), content)
self.assertEqual(0, len(config.errors))
def test_parsing(self):
config = self.parse("param1 = value1")
self.assertParameter('param1', 'value1', config.sections[0].parameters[0])
self.assertEqual(1, len(config.sections[0].parameters))
self.assertEqual(0, len(config.errors))
def test_colon_as_delimiter(self):
c = self.parse('param1 : value1')
self.assertParameter(
'param1',
'value1',
config.sections[0].parameters[0])
self.assertEqual(1, len(config.sections[0].parameters))
self.assertEqual(0, len(c.errors))
self.assertParameter('param1', 'value1', c.sections[0].parameters[0])
def test_colon_as_delimiter(self):
c = self.parse('param1 : value1')
def test_use_colon_delimiter_if_it_comes_before_equals_sign(self):
c = self.parse('param1: value=123')
self.assertEqual(0, len(c.errors))
self.assertParameter('param1', 'value=123', c.sections[0].parameters[0])
self.assertEqual(0, len(c.errors))
self.assertParameter('param1', 'value1', c.sections[0].parameters[0])
def test_use_equals_delimiter_if_it_comes_before_colon(self):
c = self.parse('param1=value:123')
self.assertEqual(0, len(c.errors))
self.assertParameter('param1', 'value:123', c.sections[0].parameters[0])
def test_use_colon_delimiter_if_it_comes_before_equals_sign(self):
c = self.parse('param1: value=123')
self.assertEqual(0, len(c.errors))
self.assertParameter(
'param1',
'value=123',
c.sections[0].parameters[0])
def test_wrapping_value_with_single_quotes(self):
c = self.parse("param = 'foo bar'")
def test_use_equals_delimiter_if_it_comes_before_colon(self):
c = self.parse('param1=value:123')
self.assertEqual(0, len(c.errors))
self.assertParameter(
'param1',
'value:123',
c.sections[0].parameters[0])
self.assertEqual(0, len(c.errors))
self.assertParameter('param', 'foo bar', c.sections[0].parameters[0])
self.assertEqual("'", c.sections[0].parameters[0].value.quotechar)
def test_wrapping_value_with_single_quotes(self):
c = self.parse("param = 'foo bar'")
def test_wrapping_value_with_single_quotes_and_trailing_whitespace(self):
c = self.parse("param = 'foo bar' ")
self.assertEqual(0, len(c.errors))
self.assertParameter('param', 'foo bar', c.sections[0].parameters[0])
self.assertEqual("'", c.sections[0].parameters[0].value.quotechar)
self.assertEqual(0, len(c.errors))
self.assertParameter('param', 'foo bar', c.sections[0].parameters[0])
def test_wrapping_value_with_single_quotes_and_trailing_whitespace(self):
c = self.parse("param = 'foo bar' ")
def test_wrapping_value_with_double_quotes(self):
c = self.parse("param = \"foo bar\"")
self.assertEqual(0, len(c.errors))
self.assertParameter('param', 'foo bar', c.sections[0].parameters[0])
self.assertEqual(0, len(c.errors))
self.assertParameter('param', 'foo bar', c.sections[0].parameters[0])
self.assertEqual('"', c.sections[0].parameters[0].value.quotechar)
def test_wrapping_value_with_double_quotes(self):
c = self.parse("param = \"foo bar\"")
def test_wrapping_value_with_double_quotes_and_trailing_whitespace(self):
c = self.parse("param = \"foo bar\" ")
self.assertEqual(0, len(c.errors))
self.assertParameter('param', 'foo bar', c.sections[0].parameters[0])
self.assertEqual('"', c.sections[0].parameters[0].value.quotechar)
self.assertEqual(0, len(c.errors))
self.assertParameter('param', 'foo bar', c.sections[0].parameters[0])
def test_wrapping_value_with_double_quotes_and_trailing_whitespace(self):
c = self.parse("param = \"foo bar\" ")
def test_parsing_iolike_source(self):
c = self.parse(StringIO("param1 = value1"))
self.assertEqual(0, len(c.errors))
self.assertParameter('param', 'foo bar', c.sections[0].parameters[0])
self.assertEqual(0, len(c.errors))
def test_parsing_iolike_source(self):
c = self.parse(StringIO("param1 = value1"))
self.assertParameter('param1', 'value1', c.sections[0].parameters[0])
self.assertEqual(1, len(c.sections[0].parameters))
self.assertEqual(0, len(c.errors))
def test_default_section_name(self):
c = self.parse("param1 = value1")
self.assertParameter('param1', 'value1', c.sections[0].parameters[0])
self.assertEqual(1, len(c.sections[0].parameters))
self.assertEqual('', c.sections[0].name.text)
def test_default_section_name(self):
c = self.parse("param1 = value1")
def test_parsing_with_section(self):
c = self.parse("""
self.assertEqual('', c.sections[0].name.text)
def test_parsing_with_section(self):
c = self.parse("""
[section1]
param1 = value1
""", margin=True)
self.assertEqual(0, len(c.errors))
self.assertEqual('section1', c.sections[0].name.text)
self.assertEqual(1, len(c.sections[0].parameters))
self.assertEqual(0, len(c.errors))
self.assertEqual('section1', c.sections[0].name.text)
self.assertEqual(1, len(c.sections[0].parameters))
def test_parsing_with_same_section(self):
c = self.parse("""
def test_parsing_with_same_section(self):
c = self.parse("""
[section1]
param1 = value1
param2 = value2
""", margin=True)
self.assertEqual(0, len(c.errors))
self.assertEqual(2, len(c.sections[0].parameters))
self.assertEqual(0, len(c.errors))
self.assertEqual(2, len(c.sections[0].parameters))
def test_parsing_with_different_sections(self):
c = self.parse("""
def test_parsing_with_different_sections(self):
c = self.parse("""
[section1]
param1 = value1
[section2]
param2 = value2
""", margin=True)
self.assertEqual(0, len(c.errors))
self.assertEqual(0, len(c.errors))
self.assertEqual('section1', c.sections[0].name.text)
self.assertParameter('param1', 'value1', c.sections[0].parameters[0])
self.assertEqual(1, len(c.sections[0].parameters))
self.assertEqual('section2', c.sections[1].name.text)
self.assertParameter('param2', 'value2', c.sections[1].parameters[0])
self.assertEqual(1, len(c.sections[1].parameters))
self.assertEqual('section1', c.sections[0].name.text)
self.assertParameter('param1', 'value1', c.sections[0].parameters[0])
self.assertEqual(1, len(c.sections[0].parameters))
self.assertEqual('section2', c.sections[1].name.text)
self.assertParameter('param2', 'value2', c.sections[1].parameters[0])
self.assertEqual(1, len(c.sections[1].parameters))
def test_whole_line_comments_starting_with_hash(self):
c = self.parse("#param=value")
self.assertEqual(0, len(c.errors))
self.assertEqual(0, len(c.sections))
def test_whole_line_comments_starting_with_hash(self):
c = self.parse("#param=value")
self.assertEqual(0, len(c.errors))
self.assertEqual(0, len(c.sections))
def test_whole_line_comments_starting_with_semicolon(self):
c = self.parse(";param=value")
self.assertEqual(0, len(c.errors))
self.assertEqual(0, len(c.sections))
def test_whole_line_comments_starting_with_semicolon(self):
c = self.parse(";param=value")
self.assertEqual(0, len(c.errors))
self.assertEqual(0, len(c.sections))
def test_hash_in_value_is_part_of_the_value(self):
c = self.parse("param=value#123")
self.assertEqual(0, len(c.errors))
self.assertParameter("param", "value#123", c.sections[0].parameters[0])
def test_hash_in_value_is_part_of_the_value(self):
c = self.parse("param=value#123")
self.assertEqual(0, len(c.errors))
self.assertParameter("param", "value#123", c.sections[0].parameters[0])
def test_multiline_value(self):
c = self.parse("""
def test_multiline_value(self):
c = self.parse("""
param1 = line1
line2
""", margin=True)
self.assertEqual(0, len(c.errors))
self.assertParameter('param1', 'line1line2', c.sections[0].parameters[0])
self.assertEqual(0, len(c.errors))
self.assertParameter(
'param1',
'line1line2',
c.sections[0].parameters[0])
def test_multiline_value_finished_by_other_parameter(self):
c = self.parse("""
def test_multiline_value_finished_by_other_parameter(self):
c = self.parse("""
param1 = foo
bar
param2 = baz
""", margin=True)
self.assertEqual(0, len(c.errors))
self.assertParameter('param1', 'foobar', c.sections[0].parameters[0])
self.assertEqual(0, len(c.errors))
self.assertParameter('param1', 'foobar', c.sections[0].parameters[0])
def test_multiline_value_finished_by_empty_line(self):
c = self.parse("""
def test_multiline_value_finished_by_empty_line(self):
c = self.parse("""
param1 = foo
bar
param2 = baz
""", margin=True)
self.assertEqual(0, len(c.errors))
self.assertParameter('param1', 'foobar', c.sections[0].parameters[0])
self.assertEqual(0, len(c.errors))
self.assertParameter('param1', 'foobar', c.sections[0].parameters[0])
def test_unclosed_section_causes_error(self):
c = self.parse("[section1\nparam1=123")
self.assertEqual(1, len(c.errors))
def test_missing_equals_sign_or_colon_causes_error(self):
c = self.parse("param1 value1")
self.assertEqual(1, len(c.errors))
def test_unclosed_section_causes_error(self):
c = self.parse("[section1\nparam1=123")
self.assertEqual(1, len(c.errors))
def test_spaces_in_key_causes_error(self):
c = self.parse("param 1 = value1")
self.assertEqual(1, len(c.errors))
def test_missing_equals_sign_or_colon_causes_error(self):
c = self.parse("param1 value1")
self.assertEqual(1, len(c.errors))
def test_returning_multiple_errors(self):
c = self.parse("[unclosed section\npararm 1 = value1")
self.assertEqual(2, len(c.errors))
def test_spaces_in_key_causes_error(self):
c = self.parse("param 1 = value1")
self.assertEqual(1, len(c.errors))
def test_errors_doesnt_affect_valid_parameters(self):
c = self.parse('param1 value1\nparam2 = value2')
self.assertEqual(1, len(c.errors))
self.assertParameter('param2', 'value2', c.sections[0].parameters[0])
def test_returning_multiple_errors(self):
c = self.parse("[unclosed section\npararm 1 = value1")
self.assertEqual(2, len(c.errors))
def test_errors_doesnt_affect_valid_parameters(self):
c = self.parse('param1 value1\nparam2 = value2')
self.assertEqual(1, len(c.errors))
self.assertParameter('param2', 'value2', c.sections[0].parameters[0])
def _getattr(self, o, name):
if name.find('.') != -1:
parts = name.split('.')
o = getattr(o, parts[0])
if o == None:
return None
return self._getattr(o, '.'.join(parts[1:]))
else:
return getattr(o, name)
def _getattr(self, o, name):
if name.find('.') != -1:
parts = name.split('.')
o = getattr(o, parts[0])
if o is None:
return None
return self._getattr(o, '.'.join(parts[1:]))
else:
return getattr(o, name)
def assertAttributes(self, attribute_values, subject):
for attr, expected in attribute_values.items():
actual = self._getattr(subject, attr)
self.assertEqual(expected, actual, "%s expected to have %s = %s, but the value was %s" % (subject, attr, expected, actual))
def assertParameter(self, name, value, o):
self.assertAttributes({'name.text': name, 'value.text': value}, o)
def assertAttributes(self, attribute_values, subject):
for attr, expected in attribute_values.items():
actual = self._getattr(subject, attr)
self.assertEqual(
expected, actual, "%s expected to have %s = %s, but the value was %s" %
(subject, attr, expected, actual))
def assertParameter(self, name, value, o):
self.assertAttributes({'name.text': name, 'value.text': value}, o)
if __name__ == '__main__':
unittest.main()
unittest.main()

387
ostack_validator/config_model.py
View File

@ -2,235 +2,306 @@ import string
from ostack_validator.common import Mark
class ConfigurationSection(object):
def __init__(self, config, section):
super(ConfigurationSection, self).__init__()
self.config = config
self.section = section
def _combine_names(self, section, param):
if section == 'DEFAULT':
return param
def __init__(self, config, section):
super(ConfigurationSection, self).__init__()
self.config = config
self.section = section
return '%s.%s' % (section, param)
def _combine_names(self, section, param):
if section == 'DEFAULT':
return param
def get(self, name, *args, **kwargs):
return self.config.get(self._combine_names(self.section, name), *args, **kwargs)
return '%s.%s' % (section, param)
def set(self, name, *args, **kwargs):
self.config.set(self._combine_names(self.section, name), *args, **kwargs)
def get(self, name, *args, **kwargs):
return (
self.config.get(
self._combine_names(
self.section,
name),
*args,
**kwargs)
)
def set_default(self, name, *args, **kwargs):
self.config.set_default(self._combine_names(self.section, name), *args, **kwargs)
def set(self, name, *args, **kwargs):
self.config.set(
self._combine_names(
self.section,
name),
*args,
**kwargs)
def contains(self, name, *args, **kwargs):
return self.config.contains(self._combine_names(self.section, name), *args, **kwargs)
def set_default(self, name, *args, **kwargs):
self.config.set_default(
self._combine_names(
self.section,
name),
*args,
**kwargs)
def is_default(self, name, *args, **kwargs):
return self.config.is_default(self._combine_names(self.section, name), *args, **kwargs)
def contains(self, name, *args, **kwargs):
return (
self.config.contains(
self._combine_names(
self.section,
name),
*args,
**kwargs)
)
def __getitem__(self, key):
return self.config.get(self._combine_names(self.section, key))
def is_default(self, name, *args, **kwargs):
return (
self.config.is_default(
self._combine_names(
self.section,
name),
*args,
**kwargs)
)
def __setitem__(self, key, value):
return self.config.set(self._combine_names(self.section, key), value)
def __getitem__(self, key):
return self.config.get(self._combine_names(self.section, key))
def __contains__(self, key):
return self.config.contains(self._combine_names(self.section, key))
def __setitem__(self, key, value):
return self.config.set(self._combine_names(self.section, key), value)
def keys(self):
return self.config.keys(self.section)
def __contains__(self, key):
return self.config.contains(self._combine_names(self.section, key))
def keys(self):
return self.config.keys(self.section)
def items(self, *args, **kwargs):
return self.config.items(self.section, *args, **kwargs)
def items(self, *args, **kwargs):
return self.config.items(self.section, *args, **kwargs)
class ConfigurationWrapper(object):
def __init__(self, config, state):
super(ConfigurationWrapper, self).__init__()
self.config = config
self.state = state
def __getitem__(self, key):
if key in self.state:
return ''
def __init__(self, config, state):
super(ConfigurationWrapper, self).__init__()
self.config = config
self.state = state
return self.config.get(key, _state=self.state)
def __getitem__(self, key):
if key in self.state:
return ''
return self.config.get(key, _state=self.state)
class Configuration(object):
def __init__(self):
super(Configuration, self).__init__()
self._defaults = dict()
self._normal = dict()
def _normalize_name(self, name):
if name.find('.') == -1:
section = 'DEFAULT'
else:
section, name = name.split('.', 1)
def __init__(self):
super(Configuration, self).__init__()
self._defaults = dict()
self._normal = dict()
return (section, name)
def _normalize_name(self, name):
if name.find('.') == -1:
section = 'DEFAULT'
else:
section, name = name.split('.', 1)
def _combine_names(self, section, param):
if section == 'DEFAULT':
return param
return (section, name)
return '%s.%s' % (section, param)
def _combine_names(self, section, param):
if section == 'DEFAULT':
return param
def get(self, name, default=None, raw=False, _state=[]):
section, name = self._normalize_name(name)
return '%s.%s' % (section, param)
if section in self._normal and name in self._normal[section]:
value = self._normal[section][name]
elif section in self._defaults and name in self._defaults[section]:
value = self._defaults[section][name]
else:
value = default
def get(self, name, default=None, raw=False, _state=[]):
section, name = self._normalize_name(name)
if not isinstance(value, str):
return value
if section in self._normal and name in self._normal[section]:
value = self._normal[section][name]
elif section in self._defaults and name in self._defaults[section]:
value = self._defaults[section][name]
else:
value = default
if raw:
return value
if not isinstance(value, str):
return value
tmpl = string.Template(value)
return tmpl.safe_substitute(ConfigurationWrapper(self, _state + [name]))
if raw:
return value
def contains(self, name, ignoreDefault=False):
section, name = self._normalize_name(name)
tmpl = string.Template(value)
return (
tmpl.safe_substitute(ConfigurationWrapper(self, _state + [name]))
)
if section in self._normal and name in self._normal[section]:
return True
def contains(self, name, ignoreDefault=False):
section, name = self._normalize_name(name)
if not ignoreDefault and section in self._defaults and name in self._defaults[section]:
return True
if section in self._normal and name in self._normal[section]:
return True
return False
if not ignoreDefault and section in self._defaults and name in self._defaults[section]:
return True
def is_default(self, name):
section, name = self._normalize_name(name)
return False
return not (section in self._normal and name in self._normal[section]) and (section in self._defaults and name in self._defaults[section])
def is_default(self, name):
section, name = self._normalize_name(name)
def set_default(self, name, value):
section, name = self._normalize_name(name)
return (
not (section in self._normal and name in self._normal[section]) and (
section in self._defaults and name in self._defaults[section])
)
if not section in self._defaults:
self._defaults[section] = dict()
def set_default(self, name, value):
section, name = self._normalize_name(name)
self._defaults[section][name] = value
if not section in self._defaults:
self._defaults[section] = dict()
def set(self, name, value):
section, name = self._normalize_name(name)
self._defaults[section][name] = value
if not section in self._normal:
self._normal[section] = dict()
def set(self, name, value):
section, name = self._normalize_name(name)
self._normal[section][name] = value
if not section in self._normal:
self._normal[section] = dict()
def section(self, section):
return ConfigurationSection(self, section)
self._normal[section][name] = value
def __getitem__(self, key):
return self.get(key)
def section(self, section):
return ConfigurationSection(self, section)
def __setitem__(self, key, value):
self.set(key, value)
def __getitem__(self, key):
return self.get(key)
def __contains__(self, section):
return (section in self._defaults) or (section in self._normal)
def __setitem__(self, key, value):
self.set(key, value)
def keys(self, section=None):
if section:
names = set()
if section in self._defaults:
for param in self._defaults[section].keys():
names.add(param)
if section in self._normal:
for param in self._normal[section].keys():
names.add(param)
def __contains__(self, section):
return (section in self._defaults) or (section in self._normal)
return list(names)
else:
sections = set()
for section in self._defaults.keys():
sections.add(section)
def keys(self, section=None):
if section:
names = set()
if section in self._defaults:
for param in self._defaults[section].keys():
names.add(param)
if section in self._normal:
for param in self._normal[section].keys():
names.add(param)
for section in self._normal.keys():
sections.add(section)
return list(names)
else:
sections = set()
for section in self._defaults.keys():
sections.add(section)
return list(sections)
for section in self._normal.keys():
sections.add(section)
def items(self, section=None):
if section:
return [(name, self.get(self._combine_names(section, name))) for name in self.keys(section)]
else:
return [(name, ConfigurationSection(self, name)) for name in self.keys()]
return list(sections)
def items(self, section=None):
if section:
return (
[(name, self.get(self._combine_names(section, name)))
for name in self.keys(section)]
)
else:
return (
[(name, ConfigurationSection(self, name))
for name in self.keys()]
)
class Element(object):
def __init__(self, start_mark, end_mark):
self.start_mark = start_mark
self.end_mark = end_mark
def __eq__(self, other):
return (self.__class__ == other.__class__) and (self.start_mark == other.start_mark) and (self.end_mark == other.end_mark)
def __init__(self, start_mark, end_mark):
self.start_mark = start_mark
self.end_mark = end_mark
def __eq__(self, other):
return (
(self.__class__ == other.__class__) and (
self.start_mark == other.start_mark) and (self.end_mark == other.end_mark)
)
def __ne__(self, other):
return not self == other
def __ne__(self, other):
return not self == other
class ComponentConfig(Element):
def __init__(self, start_mark, end_mark, name, sections=[], errors=[]):
super(ComponentConfig, self).__init__(start_mark, end_mark)
self.name = name
self.sections = sections
for section in self.sections:
section.parent = self
self.errors = errors
def __init__(self, start_mark, end_mark, name, sections=[], errors=[]):
super(ComponentConfig, self).__init__(start_mark, end_mark)
self.name = name
self.sections = sections
for section in self.sections:
section.parent = self
self.errors = errors
class TextElement(Element):
def __init__(self, start_mark, end_mark, text):
super(TextElement, self).__init__(start_mark, end_mark)
self.text = text
def __init__(self, start_mark, end_mark, text):
super(TextElement, self).__init__(start_mark, end_mark)
self.text = text
class ConfigSection(Element):
def __init__(self, start_mark, end_mark, name, parameters):
super(ConfigSection, self).__init__(start_mark, end_mark)
self.name = name
self.parameters = parameters
for parameter in self.parameters:
parameter.parent = self
class ConfigSectionName(TextElement): pass
def __init__(self, start_mark, end_mark, name, parameters):
super(ConfigSection, self).__init__(start_mark, end_mark)
self.name = name
self.parameters = parameters
for parameter in self.parameters:
parameter.parent = self
class ConfigSectionName(TextElement):
pass
class ConfigParameter(Element):
def __init__(self, start_mark, end_mark, name, value, delimiter):
super(ConfigParameter, self).__init__(start_mark, end_mark)
self.name = name
self.name.parent = self
self.value = value
self.value.parent = self
def __init__(self, start_mark, end_mark, name, value, delimiter):
super(ConfigParameter, self).__init__(start_mark, end_mark)
self.name = name
self.name.parent = self
self.delimiter = delimiter
self.delimiter.parent = self
self.value = value
self.value.parent = self
def __eq__(self, other):
return (self.name.text == other.name.text) and (self.value.text == other.value.text)
self.delimiter = delimiter
self.delimiter.parent = self
def __ne__(self, other):
return not self == other
def __repr__(self):
return "<ConfigParameter %s=%s delimiter=%s>" % (self.name.text, self.value.text, self.delimiter.text)
def __eq__(self, other):
return (
(self.name.text == other.name.text) and (
self.value.text == other.value.text)
)
def __ne__(self, other):
return not self == other
def __repr__(self):
return (
"<ConfigParameter %s=%s delimiter=%s>" % (
self.name.text,
self.value.text,
self.delimiter.text)
)
class ConfigParameterName(TextElement): pass
class ConfigParameterName(TextElement):
pass
class ConfigParameterValue(TextElement):
def __init__(self, start_mark, end_mark, text, value=None, quotechar=None):
super(ConfigParameterValue, self).__init__(start_mark, end_mark, text)
self.value = value
self.quotechar = quotechar
def __init__(self, start_mark, end_mark, text, value=None, quotechar=None):
super(ConfigParameterValue, self).__init__(start_mark, end_mark, text)
self.value = value
self.quotechar = quotechar

833
ostack_validator/discovery.py
View File

@ -11,405 +11,454 @@ from ostack_validator.model import *
class NodeClient(object):
def __init__(self, node_address, username, private_key_file, ssh_port=22):
super(NodeClient, self).__init__()
self.shell = spur.SshShell(hostname=node_address, port=ssh_port, username=username, private_key_file=private_key_file, missing_host_key=spur.ssh.MissingHostKey.accept)
def run(self, command, *args, **kwargs):
return self.shell.run(command, allow_error=True, *args, **kwargs)
def __init__(self, node_address, username, private_key_file, ssh_port=22):
super(NodeClient, self).__init__()
self.shell = spur.SshShell(
hostname=node_address,
port=ssh_port,
username=username,
private_key_file=private_key_file,
missing_host_key=spur.ssh.MissingHostKey.accept)
def open(self, path, mode='r'):
return self.shell.open(path, mode)
def run(self, command, *args, **kwargs):
return self.shell.run(command, allow_error=True, *args, **kwargs)
def open(self, path, mode='r'):
return self.shell.open(path, mode)
python_re = re.compile('(/?([^/]*/)*)python[0-9.]*')
host_port_re = re.compile('(\d+\.\d+\.\d+\.\d+):(\d+)')
class OpenstackDiscovery(object):
def discover(self, initial_nodes, username, private_key):
"Takes a list of node addresses and returns discovered openstack installation info"
openstack = Openstack()
private_key_file = None
if private_key:
private_key_file = tempfile.NamedTemporaryFile(suffix='.key')
private_key_file.write(private_key)
private_key_file.flush()
def discover(self, initial_nodes, username, private_key):
"Takes a list of node addresses and returns discovered openstack installation info"
openstack = Openstack()
for address in initial_nodes:
try:
m = host_port_re.match(address)
if m:
host = m.group(1)
port = int(m.group(2))
private_key_file = None
if private_key:
private_key_file = tempfile.NamedTemporaryFile(suffix='.key')
private_key_file.write(private_key)
private_key_file.flush()
for address in initial_nodes:
try:
m = host_port_re.match(address)
if m:
host = m.group(1)
port = int(m.group(2))
else:
host = address
port = 22
client = NodeClient(
host,
ssh_port=port,
username=username,
private_key_file=private_key_file.name)
client.run(['echo', 'test'])
except:
openstack.report_issue(
Issue(
Issue.WARNING,
"Can't connect to node %s" %
address))
continue
host = self._discover_node(client)
if len(host.components) == 0:
continue
openstack.add_host(host)
if len(openstack.hosts) == 0:
openstack.report_issue(
Issue(Issue.FATAL, "No OpenStack nodes were discovered"))
if private_key_file:
private_key_file.close()
return openstack
def _discover_node(self, client):
hostname = client.run(['hostname']).output.strip()
host = Host(name=hostname)
host.id = self._collect_host_id(client)
host.network_addresses = self._collect_host_network_addresses(client)
host.add_component(self._collect_keystone_data(client))
host.add_component(self._collect_nova_api_data(client))
host.add_component(self._collect_nova_compute_data(client))
host.add_component(self._collect_nova_scheduler_data(client))
host.add_component(self._collect_glance_api_data(client))
host.add_component(self._collect_glance_registry_data(client))
host.add_component(self._collect_cinder_api_data(client))
host.add_component(self._collect_cinder_volume_data(client))
host.add_component(self._collect_cinder_scheduler_data(client))
host.add_component(self._collect_mysql_data(client))
host.add_component(self._collect_rabbitmq_data(client))
return host
def _find_process(self, client, name):
processes = self._get_processes(client)
for line in processes:
if len(line) > 0 and os.path.basename(line[0]) == name:
return line
return None
def _find_python_process(self, client, name):
processes = self._get_processes(client)
for line in processes:
if len(line) > 0 and (line[0] == name or line[0].endswith('/' + name)):
return line
if len(line) > 1 and python_re.match(line[0]) and (line[1] == name or line[1].endswith('/' + name)):
return line
return None
def _find_python_package_version(self, client, package):
result = client.run(
['python', '-c', 'import pkg_resources; version = pkg_resources.get_provider(pkg_resources.Requirement.parse("%s")).version; print(version)' %
package])
s = result.output.strip()
parts = []
for p in s.split('.'):
if not p[0].isdigit():
break
parts.append(p)
version = '.'.join(parts)
return version
def _get_processes(self, client):
return (
[line.split()
for line in client.run(['ps', '-Ao', 'cmd', '--no-headers']).output.split("\n")]
)
def _collect_host_id(self, client):
ether_re = re.compile('link/ether (([0-9a-f]{2}:){5}([0-9a-f]{2})) ')
result = client.run(['bash', '-c', 'ip link | grep "link/ether "'])
macs = []
for match in ether_re.finditer(result.output):
macs.append(match.group(1).replace(':', ''))
return ''.join(macs)
def _collect_host_network_addresses(self, client):
ipaddr_re = re.compile('inet (\d+\.\d+\.\d+\.\d+)/\d+')
addresses = []
result = client.run(['bash', '-c', 'ip address list | grep "inet "'])
for match in ipaddr_re.finditer(result.output):
addresses.append(match.group(1))
return addresses
def _permissions_string_to_number(self, s):
return 0
def _collect_file(self, client, path):
ls = client.run(['ls', '-l', '--time-style=full-iso', path])
if ls.return_code != 0:
return None
line = ls.output.split("\n")[0]
perm, links, owner, group, size, date, time, timezone, name = line.split(
)
permissions = self._permissions_string_to_number(perm)
with client.open(path) as f:
contents = f.read()
return FileResource(path, contents, owner, group, permissions)
def _get_keystone_db_data(self, client, command, env={}):
result = client.run(['keystone', command], update_env=env)
if result.return_code != 0:
return []
lines = result.output.strip().split("\n")
columns = []
last_pos = 0
l = lines[0]
while True:
pos = l.find('+', last_pos + 1)
if pos == -1:
break
columns.append({'start': last_pos + 1, 'end': pos - 1})
last_pos = pos
l = lines[1]
for c in columns:
c['name'] = l[c['start']:c['end']].strip()
data = []
for l in lines[3:-1]:
d = dict()
for c in columns:
d[c['name']] = l[c['start']:c['end']].strip()
data.append(d)
return data
def _collect_keystone_data(self, client):
keystone_process = self._find_python_process(client, 'keystone-all')
if not keystone_process:
return None
p = index(keystone_process, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(keystone_process):
config_path = keystone_process[p + 1]
else:
host = address
port = 22
client = NodeClient(host, ssh_port=port, username=username, private_key_file=private_key_file.name)
client.run(['echo', 'test'])
except:
openstack.report_issue(Issue(Issue.WARNING, "Can't connect to node %s" % address))
continue
host = self._discover_node(client)
if len(host.components) == 0:
continue
openstack.add_host(host)
if len(openstack.hosts) == 0:
openstack.report_issue(Issue(Issue.FATAL, "No OpenStack nodes were discovered"))
if private_key_file:
private_key_file.close()
return openstack
def _discover_node(self, client):
hostname = client.run(['hostname']).output.strip()
host = Host(name=hostname)
host.id = self._collect_host_id(client)
host.network_addresses = self._collect_host_network_addresses(client)
host.add_component(self._collect_keystone_data(client))
host.add_component(self._collect_nova_api_data(client))
host.add_component(self._collect_nova_compute_data(client))
host.add_component(self._collect_nova_scheduler_data(client))
host.add_component(self._collect_glance_api_data(client))
host.add_component(self._collect_glance_registry_data(client))
host.add_component(self._collect_cinder_api_data(client))
host.add_component(self._collect_cinder_volume_data(client))
host.add_component(self._collect_cinder_scheduler_data(client))
host.add_component(self._collect_mysql_data(client))
host.add_component(self._collect_rabbitmq_data(client))
return host
def _find_process(self, client, name):
processes = self._get_processes(client)
for line in processes:
if len(line) > 0 and os.path.basename(line[0]) == name:
return line
return None
def _find_python_process(self, client, name):
processes = self._get_processes(client)
for line in processes:
if len(line) > 0 and (line[0] == name or line[0].endswith('/'+name)):
return line
if len(line) > 1 and python_re.match(line[0]) and (line[1] == name or line[1].endswith('/'+name)):
return line
return None
def _find_python_package_version(self, client, package):
result = client.run(['python', '-c', 'import pkg_resources; version = pkg_resources.get_provider(pkg_resources.Requirement.parse("%s")).version; print(version)' % package])
s = result.output.strip()
parts = []
for p in s.split('.'):
if not p[0].isdigit(): break
parts.append(p)
version = '.'.join(parts)
return version
def _get_processes(self, client):
return [line.split() for line in client.run(['ps', '-Ao', 'cmd', '--no-headers']).output.split("\n")]
def _collect_host_id(self, client):
ether_re = re.compile('link/ether (([0-9a-f]{2}:){5}([0-9a-f]{2})) ')
result = client.run(['bash', '-c', 'ip link | grep "link/ether "'])
macs = []
for match in ether_re.finditer(result.output):
macs.append(match.group(1).replace(':', ''))
return ''.join(macs)
def _collect_host_network_addresses(self, client):
ipaddr_re = re.compile('inet (\d+\.\d+\.\d+\.\d+)/\d+')
addresses = []
result = client.run(['bash', '-c', 'ip address list | grep "inet "'])
for match in ipaddr_re.finditer(result.output):
addresses.append(match.group(1))
return addresses
def _permissions_string_to_number(self, s):
return 0
def _collect_file(self, client, path):
ls = client.run(['ls', '-l', '--time-style=full-iso', path])
if ls.return_code != 0:
return None
line = ls.output.split("\n")[0]
perm, links, owner, group, size, date, time, timezone, name = line.split()
permissions = self._permissions_string_to_number(perm)
with client.open(path) as f:
contents = f.read()
return FileResource(path, contents, owner, group, permissions)
def _get_keystone_db_data(self, client, command, env={}):
result = client.run(['keystone', command], update_env=env)
if result.return_code != 0:
return []
lines = result.output.strip().split("\n")
columns = []
last_pos = 0
l = lines[0]
while True:
pos = l.find('+', last_pos+1)
if pos == -1:
break
columns.append({'start': last_pos+1, 'end': pos-1})
last_pos = pos
l = lines[1]
for c in columns:
c['name'] = l[c['start']:c['end']].strip()
data = []
for l in lines[3:-1]:
d = dict()
for c in columns:
d[c['name']] = l[c['start']:c['end']].strip()
data.append(d)
return data
def _collect_keystone_data(self, client):
keystone_process = self._find_python_process(client, 'keystone-all')
if not keystone_process:
return None
p = index(keystone_process, lambda s: s == '--config-file')
if p != -1 and p+1 < len(keystone_process):
config_path = keystone_process[p+1]
else:
config_path = '/etc/keystone/keystone.conf'
keystone = KeystoneComponent()
keystone.version = self._find_python_package_version(client, 'keystone')
keystone.config_files = []
keystone.config_files.append(self._collect_file(client, config_path))
token = keystone.config['admin_token']
host = keystone.config['bind_host']
if host == '0.0.0.0':
host = '127.0.0.1'
port = int(keystone.config['admin_port'])
keystone_env = {
'OS_SERVICE_TOKEN': token,
'OS_SERVICE_ENDPOINT': 'http://%s:%d/v2.0' % (host, port)
}
keystone.db = dict()
keystone.db['tenants'] = self._get_keystone_db_data(client, 'tenant-list', env=keystone_env)
keystone.db['users'] = self._get_keystone_db_data(client, 'user-list', env=keystone_env)
keystone.db['services'] = self._get_keystone_db_data(client, 'service-list', env=keystone_env)
keystone.db['endpoints'] = self._get_keystone_db_data(client, 'endpoint-list', env=keystone_env)
return keystone
def _collect_nova_api_data(self, client):
process = self._find_python_process(client, 'nova-api')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p+1 < len(process):
config_path = process[p+1]
else:
config_path = '/etc/nova/nova.conf'
nova_api = NovaApiComponent()
nova_api.version = self._find_python_package_version(client, 'nova')
nova_api.config_files = []
nova_api.config_files.append(self._collect_file(client, config_path))
paste_config_path = path_relative_to(nova_api.config['api_paste_config'], os.path.dirname(config_path))
nova_api.paste_config_file = self._collect_file(client, paste_config_path)
return nova_api
def _collect_nova_compute_data(self, client):
process = self._find_python_process(client, 'nova-compute')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p+1 < len(process):
config_path = process[p+1]
else:
config_path = '/etc/nova/nova.conf'
nova_compute = NovaComputeComponent()
nova_compute.version = self._find_python_package_version(client, 'nova')
nova_compute.config_files = []
nova_compute.config_files.append(self._collect_file(client, config_path))
return nova_compute
def _collect_nova_scheduler_data(self, client):
process = self._find_python_process(client, 'nova-scheduler')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p+1 < len(process):
config_path = process[p+1]
else:
config_path = '/etc/nova/nova.conf'
nova_scheduler = NovaSchedulerComponent()
nova_scheduler.version = self._find_python_package_version(client, 'nova')
nova_scheduler.config_files = []
nova_scheduler.config_files.append(self._collect_file(client, config_path))
return nova_scheduler
def _collect_glance_api_data(self, client):
process = self._find_python_process(client, 'glance-api')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p+1 < len(process):
config_path = process[p+1]
else:
config_path = '/etc/glance/glance-api.conf'
glance_api = GlanceApiComponent()
glance_api.version = self._find_python_package_version(client, 'glance')
glance_api.config_files = []
glance_api.config_files.append(self._collect_file(client, config_path))
return glance_api
def _collect_glance_registry_data(self, client):
process = self._find_python_process(client, 'glance-registry')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p+1 < len(process):
config_path = process[p+1]
else:
config_path = '/etc/glance/glance-registry.conf'
glance_registry = GlanceRegistryComponent()
glance_registry.version = self._find_python_package_version(client, 'glance')
glance_registry.config_files = []
glance_registry.config_files.append(self._collect_file(client, config_path))
return glance_registry
def _collect_cinder_api_data(self, client):
process = self._find_python_process(client, 'cinder-api')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p+1 < len(process):
config_path = process[p+1]
else:
config_path = '/etc/cinder/cinder.conf'
cinder_api = CinderApiComponent()
cinder_api.version = self._find_python_package_version(client, 'cinder')
cinder_api.config_files = []
cinder_api.config_files.append(self._collect_file(client, config_path))
paste_config_path = path_relative_to(cinder_api.config['api_paste_config'], os.path.dirname(config_path))
cinder_api.paste_config_file = self._collect_file(client, paste_config_path)
return cinder_api
def _collect_cinder_volume_data(self, client):
process = self._find_python_process(client, 'cinder-volume')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p+1 < len(process):
config_path = process[p+1]
else:
config_path = '/etc/cinder/cinder.conf'
cinder_volume = CinderVolumeComponent()
cinder_volume.version = self._find_python_package_version(client, 'cinder')
cinder_volume.config_files = []
cinder_volume.config_files.append(self._collect_file(client, config_path))
rootwrap_config_path = path_relative_to(cinder_volume.config['rootwrap_config'], os.path.dirname(config_path))
cinder_volume.rootwrap_config = self._collect_file(client, rootwrap_config_path)
return cinder_volume
def _collect_cinder_scheduler_data(self, client):
process = self._find_python_process(client, 'cinder-scheduler')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p+1 < len(process):
config_path = process[p+1]
else:
config_path = '/etc/cinder/cinder.conf'
cinder_scheduler = CinderSchedulerComponent()
cinder_scheduler.version = self._find_python_package_version(client, 'cinder')
cinder_scheduler.config_files = []
cinder_scheduler.config_files.append(self._collect_file(client, config_path))
return cinder_scheduler
def _collect_mysql_data(self, client):
process = self._find_process(client, 'mysqld')
if not process:
return None
mysqld_version_re = re.compile('mysqld\s+Ver\s(\S+)\s')
mysql = MysqlComponent()
version_result = client.run(['mysqld', '--version'])
m = mysqld_version_re.match(version_result.output)
mysql.version = m.group(1) if m else 'unknown'
mysql.config_files = []
config_locations_result = client.run(['bash', '-c', 'mysqld --help --verbose | grep "Default options are read from the following files in the given order" -A 1'])
config_locations = config_locations_result.output.strip().split("\n")[-1].split()
for path in config_locations:
f = self._collect_file(client, path)
if f:
mysql.config_files.append(f)
return mysql
def _collect_rabbitmq_data(self, client):
process = self._find_process(client, 'beam.smp')
if not process:
return None
if ' '.join(process).find('rabbit') == -1:
return None
rabbitmq = RabbitMqComponent()
rabbitmq.version = 'unknown'
return rabbitmq
config_path = '/etc/keystone/keystone.conf'
keystone = KeystoneComponent()
keystone.version = self._find_python_package_version(
client, 'keystone')
keystone.config_files = []
keystone.config_files.append(self._collect_file(client, config_path))
token = keystone.config['admin_token']
host = keystone.config['bind_host']
if host == '0.0.0.0':
host = '127.0.0.1'
port = int(keystone.config['admin_port'])
keystone_env = {
'OS_SERVICE_TOKEN': token,
'OS_SERVICE_ENDPOINT': 'http://%s:%d/v2.0' % (host, port)
}
keystone.db = dict()
keystone.db['tenants'] = self._get_keystone_db_data(
client, 'tenant-list', env=keystone_env)
keystone.db['users'] = self._get_keystone_db_data(
client, 'user-list', env=keystone_env)
keystone.db['services'] = self._get_keystone_db_data(
client, 'service-list', env=keystone_env)
keystone.db['endpoints'] = self._get_keystone_db_data(
client, 'endpoint-list', env=keystone_env)
return keystone
def _collect_nova_api_data(self, client):
process = self._find_python_process(client, 'nova-api')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(process):
config_path = process[p + 1]
else:
config_path = '/etc/nova/nova.conf'
nova_api = NovaApiComponent()
nova_api.version = self._find_python_package_version(client, 'nova')
nova_api.config_files = []
nova_api.config_files.append(self._collect_file(client, config_path))
paste_config_path = path_relative_to(
nova_api.config['api_paste_config'],
os.path.dirname(config_path))
nova_api.paste_config_file = self._collect_file(
client, paste_config_path)
return nova_api
def _collect_nova_compute_data(self, client):
process = self._find_python_process(client, 'nova-compute')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(process):
config_path = process[p + 1]
else:
config_path = '/etc/nova/nova.conf'
nova_compute = NovaComputeComponent()
nova_compute.version = self._find_python_package_version(
client, 'nova')
nova_compute.config_files = []
nova_compute.config_files.append(
self._collect_file(client, config_path))
return nova_compute
def _collect_nova_scheduler_data(self, client):
process = self._find_python_process(client, 'nova-scheduler')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(process):
config_path = process[p + 1]
else:
config_path = '/etc/nova/nova.conf'
nova_scheduler = NovaSchedulerComponent()
nova_scheduler.version = self._find_python_package_version(
client, 'nova')
nova_scheduler.config_files = []
nova_scheduler.config_files.append(
self._collect_file(client, config_path))
return nova_scheduler
def _collect_glance_api_data(self, client):
process = self._find_python_process(client, 'glance-api')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(process):
config_path = process[p + 1]
else:
config_path = '/etc/glance/glance-api.conf'
glance_api = GlanceApiComponent()
glance_api.version = self._find_python_package_version(
client, 'glance')
glance_api.config_files = []
glance_api.config_files.append(self._collect_file(client, config_path))
return glance_api
def _collect_glance_registry_data(self, client):
process = self._find_python_process(client, 'glance-registry')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(process):
config_path = process[p + 1]
else:
config_path = '/etc/glance/glance-registry.conf'
glance_registry = GlanceRegistryComponent()
glance_registry.version = self._find_python_package_version(
client, 'glance')
glance_registry.config_files = []
glance_registry.config_files.append(
self._collect_file(client, config_path))
return glance_registry
def _collect_cinder_api_data(self, client):
process = self._find_python_process(client, 'cinder-api')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(process):
config_path = process[p + 1]
else:
config_path = '/etc/cinder/cinder.conf'
cinder_api = CinderApiComponent()
cinder_api.version = self._find_python_package_version(
client, 'cinder')
cinder_api.config_files = []
cinder_api.config_files.append(self._collect_file(client, config_path))
paste_config_path = path_relative_to(
cinder_api.config['api_paste_config'],
os.path.dirname(config_path))
cinder_api.paste_config_file = self._collect_file(
client, paste_config_path)
return cinder_api
def _collect_cinder_volume_data(self, client):
process = self._find_python_process(client, 'cinder-volume')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(process):
config_path = process[p + 1]
else:
config_path = '/etc/cinder/cinder.conf'
cinder_volume = CinderVolumeComponent()
cinder_volume.version = self._find_python_package_version(
client, 'cinder')
cinder_volume.config_files = []
cinder_volume.config_files.append(
self._collect_file(client, config_path))
rootwrap_config_path = path_relative_to(
cinder_volume.config['rootwrap_config'],
os.path.dirname(config_path))
cinder_volume.rootwrap_config = self._collect_file(
client, rootwrap_config_path)
return cinder_volume
def _collect_cinder_scheduler_data(self, client):
process = self._find_python_process(client, 'cinder-scheduler')
if not process:
return None
p = index(process, lambda s: s == '--config-file')
if p != -1 and p + 1 < len(process):
config_path = process[p + 1]
else:
config_path = '/etc/cinder/cinder.conf'
cinder_scheduler = CinderSchedulerComponent()
cinder_scheduler.version = self._find_python_package_version(
client, 'cinder')
cinder_scheduler.config_files = []
cinder_scheduler.config_files.append(
self._collect_file(client, config_path))
return cinder_scheduler
def _collect_mysql_data(self, client):
process = self._find_process(client, 'mysqld')
if not process:
return None
mysqld_version_re = re.compile('mysqld\s+Ver\s(\S+)\s')
mysql = MysqlComponent()
version_result = client.run(['mysqld', '--version'])
m = mysqld_version_re.match(version_result.output)
mysql.version = m.group(1) if m else 'unknown'
mysql.config_files = []
config_locations_result = client.run(
['bash', '-c', 'mysqld --help --verbose | grep "Default options are read from the following files in the given order" -A 1'])
config_locations = config_locations_result.output.strip().split(
"\n")[-1].split()
for path in config_locations:
f = self._collect_file(client, path)
if f:
mysql.config_files.append(f)
return mysql
def _collect_rabbitmq_data(self, client):
process = self._find_process(client, 'beam.smp')
if not process:
return None
if ' '.join(process).find('rabbit') == -1:
return None
rabbitmq = RabbitMqComponent()
rabbitmq.version = 'unknown'
return rabbitmq

1
ostack_validator/inspections/__init__.py
View File

@ -1,4 +1,3 @@
from ostack_validator.inspections.keystone_authtoken import KeystoneAuthtokenSettingsInspection
from ostack_validator.inspections.keystone_endpoints import KeystoneEndpointsInspection
from ostack_validator.inspections.lettuce_runner import LettuceRunnerInspection

178
ostack_validator/inspections/keystone_authtoken.py
View File

@ -3,81 +3,137 @@ from ostack_validator.common import Inspection, Issue, find
KEYSTONE_AUTHTOKEN_FILTER_FACTORY = 'keystoneclient.middleware.auth_token:filter_factory'
class KeystoneAuthtokenSettingsInspection(Inspection):
name = 'Keystone auth'
description = 'Validate correctness of keystone settings'
name = 'Keystone auth'
description = 'Validate correctness of keystone settings'
def inspect(self, openstack):
components = []
for host in openstack.hosts:
components.extend(host.components)
def inspect(self, openstack):
components = []
for host in openstack.hosts:
components.extend(host.components)
keystones = [c for c in components if c.name == 'keystone']
if len(keystones) == 0:
openstack.report_issue(Issue(Issue.FATAL, 'No keystone service found'))
return
keystones = [c for c in components if c.name == 'keystone']
if len(keystones) == 0:
openstack.report_issue(
Issue(Issue.FATAL, 'No keystone service found'))
return
keystone = keystones[0]
keystone_addresses = [keystone.config['bind_host']]
if keystone_addresses == ['0.0.0.0']:
keystone_addresses = keystone.host.network_addresses
keystone = keystones[0]
keystone_addresses = [keystone.config['bind_host']]
if keystone_addresses == ['0.0.0.0']:
keystone_addresses = keystone.host.network_addresses
for nova in [c for c in components if c.name == 'nova-api']:
if nova.config['auth_strategy'] != 'keystone':
continue
for nova in [c for c in components if c.name == 'nova-api']:
if nova.config['auth_strategy'] != 'keystone':
continue
(authtoken_section,_) = find(
nova.paste_config.items(),
lambda (name, values): name.startswith('filter:') and values.get('paste.filter_factory') == KEYSTONE_AUTHTOKEN_FILTER_FACTORY
)
(authtoken_section, _) = find(
nova.paste_config.items(),
lambda name_values: name_values[0].startswith('filter:') and name_values[
1].get('paste.filter_factory') == KEYSTONE_AUTHTOKEN_FILTER_FACTORY
)
if not authtoken_section: continue
if not authtoken_section:
continue
authtoken_settings = nova.paste_config.section(authtoken_section)
authtoken_settings = nova.paste_config.section(authtoken_section)
def get_value(name):
return (
authtoken_settings[
name] or nova.config[
'keystone_authtoken.%s' %
name]
)
def get_value(name):
return authtoken_settings[name] or nova.config['keystone_authtoken.%s' % name]
auth_host = get_value('auth_host')
auth_port = get_value('auth_port')
auth_protocol = get_value('auth_protocol')
admin_user = get_value('admin_user')
admin_password = get_value('admin_password')
admin_tenant_name = get_value('admin_tenant_name')
admin_token = get_value('admin_token')
auth_host = get_value('auth_host')
auth_port = get_value('auth_port')
auth_protocol = get_value('auth_protocol')
admin_user = get_value('admin_user')
admin_password = get_value('admin_password')
admin_tenant_name = get_value('admin_tenant_name')
admin_token = get_value('admin_token')
msg_prefix = 'Service "%s" on host "%s"' % (
nova.name, nova.host.name)
msg_prefix = 'Service "%s" on host "%s"' % (nova.name, nova.host.name)
if not auth_host:
nova.report_issue(
Issue(
Issue.ERROR,
msg_prefix +
' miss "auth_host" setting in keystone authtoken config'))
elif not auth_host in keystone_addresses:
nova.report_issue(
Issue(
Issue.ERROR,
msg_prefix +
' has incorrect "auth_host" setting in keystone authtoken config'))
if not auth_host:
nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' miss "auth_host" setting in keystone authtoken config'))
elif not auth_host in keystone_addresses:
nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' has incorrect "auth_host" setting in keystone authtoken config'))
if not auth_port:
nova.report_issue(
Issue(
Issue.ERROR,
msg_prefix +
' miss "auth_port" setting in keystone authtoken config'))
elif auth_port != keystone.config['admin_port']:
nova.report_issue(
Issue(
Issue.ERROR,
msg_prefix +
' has incorrect "auth_port" setting in keystone authtoken config'))
if not auth_port:
nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' miss "auth_port" setting in keystone authtoken config'))
elif auth_port != keystone.config['admin_port']:
nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' has incorrect "auth_port" setting in keystone authtoken config'))
if not auth_protocol:
nova.report_issue(
Issue(
Issue.ERROR,
msg_prefix +
' miss "auth_protocol" setting in keystone authtoken config'))
elif not auth_protocol in ['http', 'https']:
nova.report_issue(
Issue(
Issue.ERROR,
msg_prefix +
' has incorrect "auth_protocol" setting in keystone authtoken config'))
if not auth_protocol:
nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' miss "auth_protocol" setting in keystone authtoken config'))
elif not auth_protocol in ['http', 'https']:
nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' has incorrect "auth_protocol" setting in keystone authtoken config'))
if not admin_user:
nova.report_issue(
Issue(
Issue.ERROR,
msg_prefix +
' miss "admin_user" setting in keystone authtoken config'))
else:
user = find(
keystone.db['users'],
lambda u: u['name'] == admin_user)
if not user:
nova.report_issue(
Issue(
Issue.ERROR,
msg_prefix +
' has "admin_user" that is missing in Keystone catalog'))
if not admin_user:
nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' miss "admin_user" setting in keystone authtoken config'))
else:
user = find(keystone.db['users'], lambda u: u['name'] == admin_user)
if not user:
nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' has "admin_user" that is missing in Keystone catalog'))
if not admin_tenant_name:
nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' miss "admin_tenant_name" setting in keystone authtoken config'))
else:
tenant = find(keystone.db['tenants'], lambda t: t['name'] == admin_tenant_name)
if not tenant:
nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' has "admin_tenant_name" that is missing in Keystone catalog'))
if admin_token:
nova.report_issue(Issue(Issue.WARNING, msg_prefix + ' uses insecure admin_token for authentication'))
if not admin_tenant_name:
nova.report_issue(
Issue(
Issue.ERROR,
msg_prefix +
' miss "admin_tenant_name" setting in keystone authtoken config'))
else:
tenant = find(
keystone.db['tenants'],
lambda t: t['name'] == admin_tenant_name)
if not tenant:
nova.report_issue(
Issue(
Issue.ERROR,
msg_prefix +
' has "admin_tenant_name" that is missing in Keystone catalog'))
if admin_token:
nova.report_issue(
Issue(
Issue.WARNING,
msg_prefix +
' uses insecure admin_token for authentication'))

71
ostack_validator/inspections/keystone_endpoints.py
View File

@ -2,40 +2,53 @@ from urlparse import urlparse
from ostack_validator.common import Inspection, Issue, find
class KeystoneEndpointsInspection(Inspection):
name = 'Keystone endpoints'
description = 'Validate that each keystone endpoint leads to proper service'
name = 'Keystone endpoints'
description = 'Validate that each keystone endpoint leads to proper service'
def inspect(self, openstack):
keystone = find(openstack.components, lambda c: c.name == 'keystone')
if not keystone:
return
def inspect(self, openstack):
keystone = find(openstack.components, lambda c: c.name == 'keystone')
if not keystone:
return
for service in keystone.db['services']:
if service['type'] == 'compute':
endpoint = find(keystone.db['endpoints'], lambda e: e['service_id'] == service['id'])
if not endpoint:
keystone.report_issue(Issue(Issue.WARNING, 'Keystone catalog contains service "%s" that has no defined endpoints' % service['name']))
continue
for service in keystone.db['services']:
if service['type'] == 'compute':
endpoint = find(
keystone.db['endpoints'],
lambda e: e['service_id'] == service['id'])
if not endpoint:
keystone.report_issue(
Issue(
Issue.WARNING, 'Keystone catalog contains service "%s" that has no defined endpoints' %
service['name']))
continue
for url_attr in ['adminurl', 'publicurl', 'internalurl']:
url = urlparse(endpoint[url_attr])
for url_attr in ['adminurl', 'publicurl', 'internalurl']:
url = urlparse(endpoint[url_attr])
# TODO: resolve endpoint url host address
host = find(openstack.hosts, lambda h: url.hostname in h.network_addresses)
if not host:
keystone.report_issue(Issue(Issue.ERROR, 'Keystone catalog has endpoint for service "%s" (id %s) that has "%s" set pointing to unknown host' % (service['name'], service['id'], url_attr)))
continue
# TODO: resolve endpoint url host address
host = find(
openstack.hosts,
lambda h: url.hostname in h.network_addresses)
if not host:
keystone.report_issue(
Issue(
Issue.ERROR, 'Keystone catalog has endpoint for service "%s" (id %s) that has "%s" set pointing to unknown host' %
(service['name'], service['id'], url_attr)))
continue
nova_compute = None
for c in host.components:
if c.name != 'nova-compute': continue
if c.config['osapi_compute_listen'] in ['0.0.0.0', url.hostname] and c.config['osapi_compute_listen_port'] == url.port:
nova_compute = c
break
if not nova_compute:
keystone.report_issue(Issue(Issue.ERROR, 'Keystone catalog has endpoint for service "%s" (id %s) that has "%s" set pointing to no service' % (service['name'], service['id'], url_attr)))
nova_compute = None
for c in host.components:
if c.name != 'nova-compute':
continue
if c.config['osapi_compute_listen'] in ['0.0.0.0', url.hostname] and c.config['osapi_compute_listen_port'] == url.port:
nova_compute = c
break
if not nova_compute:
keystone.report_issue(
Issue(
Issue.ERROR, 'Keystone catalog has endpoint for service "%s" (id %s) that has "%s" set pointing to no service' %
(service['name'], service['id'], url_attr)))

95
ostack_validator/inspections/lettuce/steps.py
View File

@ -4,77 +4,94 @@ from lettuce import *
from ostack_validator.common import Issue, Version, find
from ostack_validator.model import *
def get_variable(name):
if not hasattr(world, 'variables'):
return None
return world.variables.get(name)
def get_variable(name):
if not hasattr(world, 'variables'):
return None
return world.variables.get(name)
def set_variable(name, value):
if not hasattr(world, 'variables'):
world.variables = {}
if not hasattr(world, 'variables'):
world.variables = {}
world.variables[name] = value
world.variables[name] = value
def subst(template):
if not hasattr(world, 'variables'):
return template
if not hasattr(world, 'variables'):
return template
tmpl = string.Template(template)
return tmpl.safe_substitute(world.variables)
tmpl = string.Template(template)
return tmpl.safe_substitute(world.variables)
def stop():
assert False, "stop"
assert False, "stop"
# Openstack general step description section
@step(r'I use OpenStack (\w+)')
def use_openstack_version(step, version):
version = Version(version)
for component in [c for c in world.openstack.components if isinstance(c, OpenstackComponent)]:
if not Version(component.version) >= version: stop()
version = Version(version)
for component in [c for c in world.openstack.components if isinstance(c, OpenstackComponent)]:
if not Version(component.version) >= version:
stop()
@step(r'Controller addresses are @(\w+)')
def controller_addresses(self, variable):
controller = find(world.openstack.components, lambda c: c.name == 'nova')
controller = find(world.openstack.components, lambda c: c.name == 'nova')
if controller.config['s3_host'] == '0.0.0.0':
addresses = filter(lambda ip: not ip.startswith('127.'), controller.host.network_addresses)
else:
addresses = [controller.config['s3_host']]
if controller.config['s3_host'] == '0.0.0.0':
addresses = filter(
lambda ip: not ip.startswith('127.'),
controller.host.network_addresses)
else:
addresses = [controller.config['s3_host']]
set_variable(variable, addresses)
set_variable(variable, addresses)
# Keystone steps section
@step(r'Keystone addresses are @(\w+)')
def keystone_addresses(self, variable):
keystone = find(world.openstack.components, lambda c: c.name == 'keystone')
keystone = find(world.openstack.components, lambda c: c.name == 'keystone')
if keystone.config['bind_host'] == '0.0.0.0':
addresses = filter(lambda ip: not ip.startswith('127.'), keystone.host.network_addresses)
else:
addresses = [keystone.config['bind_host']]
if keystone.config['bind_host'] == '0.0.0.0':
addresses = filter(
lambda ip: not ip.startswith('127.'),
keystone.host.network_addresses)
else:
addresses = [keystone.config['bind_host']]
set_variable(variable, addresses)
set_variable(variable, addresses)
# Nova steps section
@step(r'Nova has "(.+)" equal to "(.*)"')
def nova_has_property(step, name, value):
name = subst(name)
value = subst(value)
name = subst(name)
value = subst(value)
for nova in [c for c in world.openstack.components if c.name.startswith('nova')]:
if not nova.config[name] == value:
stop()
for nova in [c for c in world.openstack.components if c.name.startswith('nova')]:
if not nova.config[name] == value: stop()
@step(r'Nova should have "(.+)" in "(.*)"')
def nova_property_assertion(self, name, values):
name = subst(name)
values = subst(values)
name = subst(name)
values = subst(values)
if not values:
return
if not values:
return
for nova in [c for c in world.openstack.components if c.name.startswith('nova')]:
nova_value = nova.config[name]
for nova in [c for c in world.openstack.components if c.name.startswith('nova')]:
nova_value = nova.config[name]
if not (nova_value and nova_value in values):
nova.report_issue(Issue(Issue.ERROR, 'Nova should have "%s" in %s' % (name, values)))
if not (nova_value and nova_value in values):
nova.report_issue(
Issue(Issue.ERROR, 'Nova should have "%s" in %s' %
(name, values)))

27
ostack_validator/inspections/lettuce_runner.py
View File

@ -3,18 +3,23 @@ import lettuce
from ostack_validator.common import Inspection, Issue
class LettuceRunnerInspection(Inspection):
def inspect(self, openstack):
runner = lettuce.Runner(
base_path=os.path.join(os.path.dirname(__file__), 'lettuce')
)
lettuce.world.openstack = openstack
result = runner.run()
del lettuce.world.openstack
def inspect(self, openstack):
runner = lettuce.Runner(
base_path=os.path.join(os.path.dirname(__file__), 'lettuce')
)
for feature_result in result.feature_results:
for scenario_result in [s for s in feature_result.scenario_results if not s.passed]:
for step in scenario_result.steps_undefined:
openstack.report_issue(Issue(Issue.ERROR, 'Undefined step "%s"' % step.sentence))
lettuce.world.openstack = openstack
result = runner.run()
del lettuce.world.openstack
for feature_result in result.feature_results:
for scenario_result in [s for s in feature_result.scenario_results if not s.passed]:
for step in scenario_result.steps_undefined:
openstack.report_issue(
Issue(
Issue.ERROR,
'Undefined step "%s"' %
step.sentence))

52
ostack_validator/main.py
View File

@ -5,38 +5,42 @@ import argparse
from ostack_validator.model_parser import ModelParser
from ostack_validator.inspection import MainConfigValidationInspection
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--debug', help='set debug log level', action='store_true')
parser.add_argument('path', help='Path to config snapshot')
parser = argparse.ArgumentParser()
parser.add_argument(
'-d',
'--debug',
help='set debug log level',
action='store_true')
parser.add_argument('path', help='Path to config snapshot')
args = parser.parse_args(args)
args = parser.parse_args(args)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARN)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARN)
model_parser = ModelParser()
model_parser = ModelParser()
print('Analyzing configs in "%s"' % args.path)
print('Analyzing configs in "%s"' % args.path)
model = model_parser.parse(args.path)
model = model_parser.parse(args.path)
inspections = [MainConfigValidationInspection()]
inspections = [MainConfigValidationInspection()]
issues = []
for inspection in inspections:
issues.extend(inspection.inspect(model))
issues = []
for inspection in inspections:
issues.extend(inspection.inspect(model))
if len(issues) == 0:
print('No issues found')
else:
print('Found issues:')
for issue in issues:
print(issue)
if len(issues) == 0:
print('No issues found')
else:
print('Found issues:')
for issue in issues:
print(issue)
if __name__ == '__main__':
main(sys.argv[1:])
main(sys.argv[1:])

454
ostack_validator/model.py
View File

@ -10,277 +10,341 @@ import ostack_validator.schemas
from ostack_validator.config_formats import IniConfigParser
from ostack_validator.utils import memoized
class IssueReporter(object):
def __init__(self):
super(IssueReporter, self).__init__()
self.issues = []
def report_issue(self, issue):
issue.subject = self
self.issues.append(issue)
def __init__(self):
super(IssueReporter, self).__init__()
self.issues = []
def report_issue(self, issue):
issue.subject = self
self.issues.append(issue)
@property
def all_issues(self):
return list(self.issues)
@property
def all_issues(self):
return list(self.issues)
class Openstack(IssueReporter):
def __init__(self):
super(Openstack, self).__init__()
self.hosts = []
def add_host(self, host):
if not host: return
def __init__(self):
super(Openstack, self).__init__()
self.hosts = []
self.hosts.append(host)
host.parent = self
def add_host(self, host):
if not host:
return
@property
def all_issues(self):
result = super(Openstack, self).all_issues
self.hosts.append(host)
host.parent = self
for host in self.hosts:
result.extend(host.all_issues)
@property
def all_issues(self):
result = super(Openstack, self).all_issues
return result
for host in self.hosts:
result.extend(host.all_issues)
@property
def components(self):
components = []
for host in self.hosts:
components.extend(host.components)
return result
return components
@property
def components(self):
components = []
for host in self.hosts:
components.extend(host.components)
return components
class Host(IssueReporter):
def __init__(self, name):
super(Host, self).__init__()
self.name = name
self.components = []
def __str__(self):
return 'Host "%s"' % self.name
def __init__(self, name):
super(Host, self).__init__()
self.name = name
self.components = []
def add_component(self, component):
if not component: return
def __str__(self):
return 'Host "%s"' % self.name
self.components.append(component)
component.parent = self
def add_component(self, component):
if not component:
return
@property
def openstack(self):
return self.parent
self.components.append(component)
component.parent = self
@property
def all_issues(self):
result = super(Host, self).all_issues
@property
def openstack(self):
return self.parent
for component in self.components:
result.extend(component.all_issues)
@property
def all_issues(self):
result = super(Host, self).all_issues
return result
for component in self.components:
result.extend(component.all_issues)
return result
class Service(IssueReporter):
def __init__(self):
super(Service, self).__init__()
self.issues = []
def report_issue(self, issue):
self.issues.append(issue)
def __init__(self):
super(Service, self).__init__()
self.issues = []
@property
def host(self):
return self.parent
def report_issue(self, issue):
self.issues.append(issue)
@property
def openstack(self):
return self.host.openstack
@property
def host(self):
return self.parent
@property
def all_issues(self):
result = super(Service, self).all_issues
@property
def openstack(self):
return self.host.openstack
if hasattr(self, 'config_files') and self.config_files:
[result.extend(config_file.all_issues) for config_file in self.config_files]
@property
def all_issues(self):
result = super(Service, self).all_issues
return result
if hasattr(self, 'config_files') and self.config_files:
[result.extend(config_file.all_issues)
for config_file in self.config_files]
def __str__(self):
return 'Service "%s"' % self.name
return result
def __str__(self):
return 'Service "%s"' % self.name
class OpenstackComponent(Service):
logger = logging.getLogger('ostack_validator.model.openstack_component')
component = None
logger = logging.getLogger('ostack_validator.model.openstack_component')
component = None
@property
@memoized
def config(self):
schema = ConfigSchemaRegistry.get_schema(self.component, self.version)
if not schema:
self.logger.debug('No schema for component "%s" main config version %s. Skipping it' % (self.component, self.version))
return None
@property
@memoized
def config(self):
schema = ConfigSchemaRegistry.get_schema(self.component, self.version)
if not schema:
self.logger.debug(
'No schema for component "%s" main config version %s. Skipping it' %
(self.component, self.version))
return None
return self._parse_config_resources(self.config_files, schema)
return self._parse_config_resources(self.config_files, schema)
def _parse_config_resources(self, resources, schema=None):
config = Configuration()
def _parse_config_resources(self, resources, schema=None):
config = Configuration()
# Apply defaults
if schema:
for parameter in filter(lambda p: p.default, schema.parameters):
if parameter.section == 'DEFAULT':
config.set_default(parameter.name, parameter.default)
else:
config.set_default('%s.%s' % (parameter.section, parameter.name), parameter.default)
for resource in reversed(resources):
self._parse_config_file(Mark(resource.path), resource.contents, config, schema, issue_reporter=resource)
return config
def _parse_config_file(self, base_mark, config_contents, config=Configuration(), schema=None, issue_reporter=None):
if issue_reporter:
def report_issue(issue):
issue_reporter.report_issue(issue)
else:
def report_issue(issue): pass
# Parse config file
config_parser = IniConfigParser()
parsed_config = config_parser.parse('', base_mark, config_contents)
for error in parsed_config.errors:
report_issue(error)
# Validate config parameters and store them
section_name_text_f = lambda s: s.name.text
sections_by_name = groupby(sorted(parsed_config.sections, key=section_name_text_f), key=section_name_text_f)
for section_name, sections in sections_by_name:
sections = list(sections)
if len(sections) > 1:
report_issue(Issue(Issue.INFO, 'Section "%s" appears multiple times' % section_name))
seen_parameters = set()
for section in sections:
unknown_section = False
# Apply defaults
if schema:
unknown_section = not schema.has_section(section.name.text)
for parameter in filter(lambda p: p.default, schema.parameters):
if parameter.section == 'DEFAULT':
config.set_default(parameter.name, parameter.default)
else:
config.set_default(
'%s.%s' %
(parameter.section, parameter.name), parameter.default)
if unknown_section:
report_issue(MarkedIssue(Issue.WARNING, 'Unknown section "%s"' % (section_name), section.start_mark))
continue
for resource in reversed(resources):
self._parse_config_file(
Mark(resource.path),
resource.contents,
config,
schema,
issue_reporter=resource)
for parameter in section.parameters:
parameter_schema = None
if schema:
parameter_schema = schema.get_parameter(name=parameter.name.text, section=section.name.text)
if not (parameter_schema or unknown_section):
report_issue(MarkedIssue(Issue.WARNING, 'Unknown parameter: section "%s" name "%s"' % (section_name, parameter.name.text), parameter.start_mark))
continue
return config
if parameter.name.text in seen_parameters:
report_issue(MarkedIssue(Issue.WARNING, 'Parameter "%s" in section "%s" redeclared' % (parameter.name.text, section_name), parameter.start_mark))
else:
seen_parameters.add(parameter.name.text)
def _parse_config_file(
self,
base_mark,
config_contents,
config=Configuration(),
schema=None,
issue_reporter=None):
if issue_reporter:
def report_issue(issue):
issue_reporter.report_issue(issue)
else:
def report_issue(issue):
pass
parameter_fullname = parameter.name.text
if section_name != 'DEFAULT':
parameter_fullname = section_name + '.' + parameter_fullname
# Parse config file
config_parser = IniConfigParser()
parsed_config = config_parser.parse('', base_mark, config_contents)
for error in parsed_config.errors:
report_issue(error)
if parameter_schema:
type_validator = TypeValidatorRegistry.get_validator(parameter_schema.type)
type_validation_result = type_validator.validate(parameter.value.text)
if isinstance(type_validation_result, Issue):
type_validation_result.mark = parameter.value.start_mark.merge(type_validation_result.mark)
type_validation_result.message = 'Property "%s" in section "%s": %s' % (parameter.name.text, section_name, type_validation_result.message)
report_issue(type_validation_result)
# Validate config parameters and store them
section_name_text_f = lambda s: s.name.text
sections_by_name = groupby(
sorted(
parsed_config.sections,
key=section_name_text_f),
key=section_name_text_f)
config.set(parameter_fullname, parameter.value.text)
else:
value = type_validation_result
for section_name, sections in sections_by_name:
sections = list(sections)
config.set(parameter_fullname, value)
if len(sections) > 1:
report_issue(
Issue(
Issue.INFO,
'Section "%s" appears multiple times' %
section_name))
# if value == parameter_schema.default:
# report_issue(MarkedIssue(Issue.INFO, 'Explicit value equals default: section "%s" parameter "%s"' % (section_name, parameter.name.text), parameter.start_mark))
if parameter_schema.deprecation_message:
report_issue(MarkedIssue(Issue.WARNING, 'Deprecated parameter: section "%s" name "%s". %s' % (section_name, parameter.name.text, parameter_schema.deprecation_message), parameter.start_mark))
else:
config.set(parameter_fullname, parameter.value.text)
seen_parameters = set()
return config
for section in sections:
unknown_section = False
if schema:
unknown_section = not schema.has_section(section.name.text)
if unknown_section:
report_issue(
MarkedIssue(Issue.WARNING, 'Unknown section "%s"' %
(section_name), section.start_mark))
continue
for parameter in section.parameters:
parameter_schema = None
if schema:
parameter_schema = schema.get_parameter(
name=parameter.name.text,
section=section.name.text)
if not (parameter_schema or unknown_section):
report_issue(
MarkedIssue(
Issue.WARNING, 'Unknown parameter: section "%s" name "%s"' %
(section_name, parameter.name.text), parameter.start_mark))
continue
if parameter.name.text in seen_parameters:
report_issue(
MarkedIssue(
Issue.WARNING, 'Parameter "%s" in section "%s" redeclared' %
(parameter.name.text, section_name), parameter.start_mark))
else:
seen_parameters.add(parameter.name.text)
parameter_fullname = parameter.name.text
if section_name != 'DEFAULT':
parameter_fullname = section_name + \
'.' + parameter_fullname
if parameter_schema:
type_validator = TypeValidatorRegistry.get_validator(
parameter_schema.type)
type_validation_result = type_validator.validate(
parameter.value.text)
if isinstance(type_validation_result, Issue):
type_validation_result.mark = parameter.value.start_mark.merge(
type_validation_result.mark)
type_validation_result.message = 'Property "%s" in section "%s": %s' % (
parameter.name.text, section_name, type_validation_result.message)
report_issue(type_validation_result)
config.set(
parameter_fullname,
parameter.value.text)
else:
value = type_validation_result
config.set(parameter_fullname, value)
# if value == parameter_schema.default:
# report_issue(MarkedIssue(Issue.INFO, 'Explicit value equals default: section "%s" parameter "%s"' % (section_name, parameter.name.text), parameter.start_mark))
if parameter_schema.deprecation_message:
report_issue(
MarkedIssue(
Issue.WARNING, 'Deprecated parameter: section "%s" name "%s". %s' %
(section_name, parameter.name.text, parameter_schema.deprecation_message), parameter.start_mark))
else:
config.set(parameter_fullname, parameter.value.text)
return config
class KeystoneComponent(OpenstackComponent):
component = 'keystone'
name = 'keystone'
component = 'keystone'
name = 'keystone'
class GlanceApiComponent(OpenstackComponent):
component = 'glance'
name = 'glance-api'
component = 'glance'
name = 'glance-api'
class GlanceRegistryComponent(OpenstackComponent):
component = 'glance'
name = 'glance-registry'
component = 'glance'
name = 'glance-registry'
class NovaApiComponent(OpenstackComponent):
component = 'nova'
name = 'nova-api'
component = 'nova'
name = 'nova-api'
@property
@memoized
def paste_config(self):
return self._parse_config_resources([self.paste_config_file])
@property
@memoized
def paste_config(self):
return self._parse_config_resources([self.paste_config_file])
@property
def all_issues(self):
result = super(NovaApiComponent, self).all_issues
@property
def all_issues(self):
result = super(NovaApiComponent, self).all_issues
if hasattr(self, 'paste_config_file') and self.paste_config_file:
result.extend(self.paste_config_file.all_issues)
return result
if hasattr(self, 'paste_config_file') and self.paste_config_file:
result.extend(self.paste_config_file.all_issues)
return result
class NovaComputeComponent(OpenstackComponent):
component = 'nova'
name = 'nova-compute'
component = 'nova'
name = 'nova-compute'
class NovaSchedulerComponent(OpenstackComponent):
component = 'nova'
name = 'nova-scheduler'
component = 'nova'
name = 'nova-scheduler'
class CinderApiComponent(OpenstackComponent):
component = 'cinder'
name = 'cinder-api'
component = 'cinder'
name = 'cinder-api'
class CinderVolumeComponent(OpenstackComponent):
component = 'cinder'
name = 'cinder-volume'
component = 'cinder'
name = 'cinder-volume'
class CinderSchedulerComponent(OpenstackComponent):
component = 'cinder'
name = 'cinder-scheduler'
component = 'cinder'
name = 'cinder-scheduler'
class MysqlComponent(Service):
component = 'mysql'
name = 'mysql'
component = 'mysql'
name = 'mysql'
class RabbitMqComponent(Service):
name = 'rabbitmq'
name = 'rabbitmq'
class FileResource(IssueReporter):
def __init__(self, path, contents, owner, group, permissions):
super(FileResource, self).__init__()
self.path = path
self.contents = contents
self.owner = owner
self.group = group
self.permissions = permissions
def __str__(self):
return 'File "%s"' % self.path
def __init__(self, path, contents, owner, group, permissions):
super(FileResource, self).__init__()
self.path = path
self.contents = contents
self.owner = owner
self.group = group
self.permissions = permissions
def __str__(self):
return 'File "%s"' % self.path

285
ostack_validator/resource.py
View File

@ -4,162 +4,205 @@ import json
from ostack_validator.common import Error, Version
class Resource(object):
HOST = 'host'
FILE = 'file'
DIRECTORY = 'directory'
SERVICE = 'service'
HOST = 'host'
FILE = 'file'
DIRECTORY = 'directory'
SERVICE = 'service'
def __init__(self, name):
super(Resource, self).__init__()
self.name = name
def __init__(self, name):
super(Resource, self).__init__()
self.name = name
def __str__(self):
return self.name
def __str__(self):
return self.name
def __repr__(self):
return '<%s name=%s>' % (str(self.__class__).split('.')[-1], self.name)
def __repr__(self):
return '<%s name=%s>' % (str(self.__class__).split('.')[-1], self.name)
def get_contents(self):
raise Error('Not implemented')
def get_contents(self):
raise Error, 'Not implemented'
class ResourceLocator(object):
def find_resource(self, resource_type, name, host=None, **kwargs):
return None
def find_resource(self, resource_type, name, host=None, **kwargs):
return None
class HostResource(Resource):
def __init__(self, name, resource_locator, interfaces=[]):
super(HostResource, self).__init__(name)
self.resource_locator = resource_locator
self.interfaces = interfaces
def find_resource(self, resource_type, name=None, **kwargs):
return self.resource_locator.find_resource(resource_type, name, host=self, **kwargs)
def __init__(self, name, resource_locator, interfaces=[]):
super(HostResource, self).__init__(name)
self.resource_locator = resource_locator
self.interfaces = interfaces
def find_resource(self, resource_type, name=None, **kwargs):
return (
self.resource_locator.find_resource(
resource_type,
name,
host=self,
**kwargs)
)
class DirectoryResource(Resource):
def __init__(self, name, owner=None, group=None, permissions=None):
super(DirectoryResource, self).__init__(name)
self.owner = owner
self.group = group
self.permissions = permissions
def __init__(self, name, owner=None, group=None, permissions=None):
super(DirectoryResource, self).__init__(name)
self.owner = owner
self.group = group
self.permissions = permissions
class FileResource(Resource):
def __init__(self, name, path, owner=None, group=None, permissions=None):
super(FileResource, self).__init__(name)
self.path = path
self.owner = owner
self.group = group
self.permissions = permissions
def get_contents(self):
with open(self.path) as f:
return f.read()
def __init__(self, name, path, owner=None, group=None, permissions=None):
super(FileResource, self).__init__(name)
self.path = path
self.owner = owner
self.group = group
self.permissions = permissions
def get_contents(self):
with open(self.path) as f:
return f.read()
class ServiceResource(Resource):
def __init__(self, name, version, metadata={}):
super(ServiceResource, self).__init__(name)
self.version = Version(version)
self.metadata = metadata
def __init__(self, name, version, metadata={}):
super(ServiceResource, self).__init__(name)
self.version = Version(version)
self.metadata = metadata
class FilesystemSnapshot(object):
def __init__(self, path):
super(FilesystemSnapshot, self).__init__()
self.path = path
self.basedir = os.path.join(os.path.dirname(self.path), 'root')
self._parse_snapshot()
def get_resource(self, path):
if path in self._resources:
return self._resources[path]
def __init__(self, path):
super(FilesystemSnapshot, self).__init__()
self.path = path
self.basedir = os.path.join(os.path.dirname(self.path), 'root')
self._parse_snapshot()
return None
def get_resource(self, path):
if path in self._resources:
return self._resources[path]
def _parse_snapshot(self):
self._resources = {}
if not os.path.isfile(self.path): return
with open(self.path) as f:
for line in f.readlines():
line = line.lstrip()
if line == '' or line.startswith('#'): continue
return None
def _parse_snapshot(self):
self._resources = {}
if not os.path.isfile(self.path):
return
with open(self.path) as f:
for line in f.readlines():
line = line.lstrip()
if line == '' or line.startswith('#'):
continue
resource_type = line.split('|')[0]
if resource_type == 'dir':
source_path, owner, group, permissions = line.split(
'|')[1:]
self._resources[source_path] = DirectoryResource(
source_path,
owner=owner,
group=group,
permissions=permissions)
elif resource_type == 'file':
source_path, local_path, owner, group, permissions = line.split(
'|')[1:]
self._resources[source_path] = FileResource(
os.path.basename(source_path),
path=os.path.join(self.basedir,
local_path),
owner=owner,
group=group,
permissions=permissions)
else:
self.logger.warn(
'Unknown resource "%s" in line "%s"' %
(resource_type, line))
resource_type = line.split('|')[0]
if resource_type == 'dir':
source_path, owner, group, permissions = line.split('|')[1:]
self._resources[source_path] = DirectoryResource(source_path, owner=owner, group=group, permissions=permissions)
elif resource_type == 'file':
source_path, local_path, owner, group, permissions = line.split('|')[1:]
self._resources[source_path] = FileResource(os.path.basename(source_path), path=os.path.join(self.basedir, local_path), owner=owner, group=group, permissions=permissions)
else:
self.logger.warn('Unknown resource "%s" in line "%s"' % (resource_type, line))
class ConfigSnapshotResourceLocator(object):
def __init__(self, basedir):
super(ConfigSnapshotResourceLocator, self).__init__()
self.basedir = basedir
if not os.path.isdir(self.basedir):
raise Error, 'Invalid argument: base directory does not exist'
self._services = None
self._filesystem_snapshots = {}
def find_resource(self, resource_type, name=None, host=None, **kwargs):
if resource_type == Resource.HOST:
if name:
host_path = os.path.join(self.basedir, name)
if not os.path.isdir(host_path):
return None
return HostResource(name, self)
else:
return [HostResource(os.path.basename(host_path), self) for host_path in glob.glob(os.path.join(self.basedir, '*')) if os.path.isdir(host_path)]
if resource_type == Resource.FILE:
if not host:
raise Error, 'Invalid argument: "host" need to be specified'
def __init__(self, basedir):
super(ConfigSnapshotResourceLocator, self).__init__()
self.basedir = basedir
if not os.path.isdir(self.basedir):
raise Error('Invalid argument: base directory does not exist')
self._services = None
self._filesystem_snapshots = {}
if isinstance(host, HostResource):
host = host.name
def find_resource(self, resource_type, name=None, host=None, **kwargs):
if resource_type == Resource.HOST:
if name:
host_path = os.path.join(self.basedir, name)
if not os.path.isdir(host_path):
return None
return HostResource(name, self)
else:
return (
[HostResource(os.path.basename(host_path), self)
for host_path in glob.glob(os.path.join(self.basedir, '*')) if os.path.isdir(host_path)]
)
if resource_type == Resource.FILE:
if not host:
raise Error('Invalid argument: "host" need to be specified')
if name:
return self._get_filesystem_snapshot(host).get_resource(name)
else:
return []
elif resource_type == Resource.SERVICE:
if not host:
raise Error, 'Invalid argument: "host" need to be specified'
if isinstance(host, HostResource):
host = host.name
if isinstance(host, HostResource):
host = host.name
if name:
return self._get_filesystem_snapshot(host).get_resource(name)
else:
return []
elif resource_type == Resource.SERVICE:
if not host:
raise Error('Invalid argument: "host" need to be specified')
self._ensure_services_loaded()
if isinstance(host, HostResource):
host = host.name
if name:
if name in self._services:
return self._services[host][name]
self._ensure_services_loaded()
if name:
if name in self._services:
return self._services[host][name]
else:
return None
else:
return self._services[host].values()
else:
return None
else:
return self._services[host].values()
else:
return None
return None
def _ensure_services_loaded(self):
if self._services: return
def _ensure_services_loaded(self):
if self._services:
return
self._services = {}
for host_path in glob.glob(os.path.join(self.basedir, '*')):
if not os.path.isdir(host_path): continue
self._services = {}
for host_path in glob.glob(os.path.join(self.basedir, '*')):
if not os.path.isdir(host_path):
continue
services_json_path = os.path.join(host_path, 'services.json')
if not os.path.isfile(services_json_path): continue
services_json_path = os.path.join(host_path, 'services.json')
if not os.path.isfile(services_json_path):
continue
host_name = os.path.basename(host_path)
self._services[host_name] = {}
with open(services_json_path) as f:
for service_name, metadata in json.loads(f.read()).items():
version = metadata.pop('version')
self._services[host_name][service_name] = ServiceResource(service_name, str(version), metadata)
def _get_filesystem_snapshot(self, host):
if not host in self._filesystem_snapshots:
self._filesystem_snapshots[host] = FilesystemSnapshot(os.path.join(self.basedir, host, 'filesystem'))
return self._filesystem_snapshots[host]
host_name = os.path.basename(host_path)
self._services[host_name] = {}
with open(services_json_path) as f:
for service_name, metadata in json.loads(f.read()).items():
version = metadata.pop('version')
self._services[host_name][service_name] = ServiceResource(
service_name, str(version), metadata)
def _get_filesystem_snapshot(self, host):
if not host in self._filesystem_snapshots:
self._filesystem_snapshots[host] = FilesystemSnapshot(
os.path.join(self.basedir, host, 'filesystem'))
return self._filesystem_snapshots[host]

685
ostack_validator/schema.py
View File

@ -2,415 +2,528 @@ import sys
from ostack_validator.common import Issue, MarkedIssue, Mark, Version, find, index
class SchemaUpdateRecord(object):
# checkpoint's data is version number
def __init__(self, version, operation, data=None):
super(SchemaUpdateRecord, self).__init__()
if not operation in ['checkpoint', 'add', 'remove']:
raise Error, 'Unknown operation "%s"' % operation
version = Version(version)
self.version = version
self.operation = operation
self.data = data
def __repr__(self):
return '<SchemaUpdateRecord %s %s %s' % (self.version, self.operation, self.data)
class SchemaUpdateRecord(object):
# checkpoint's data is version number
def __init__(self, version, operation, data=None):
super(SchemaUpdateRecord, self).__init__()
if not operation in ['checkpoint', 'add', 'remove']:
raise Error('Unknown operation "%s"' % operation)
version = Version(version)
self.version = version
self.operation = operation
self.data = data
def __repr__(self):
return (
'<SchemaUpdateRecord %s %s %s' % (
self.version,
self.operation,
self.data)
)
class SchemaBuilder(object):
def __init__(self, name, data):
super(SchemaBuilder, self).__init__()
self.name = name
self.data = data
self.current_version = None
self.current_section = None
self.adds = []
self.removals = []
def __init__(self, name, data):
super(SchemaBuilder, self).__init__()
self.name = name
self.data = data
def __del__(self):
if len(self.adds) > 0 or len(self.removals) > 0:
sys.stderr.write("WARNING: Uncommitted config schema \"%s\" version %s\n" % (self.name, self.current_version))
self.current_version = None
self.current_section = None
self.adds = []
self.removals = []
def version(self, version, checkpoint=False):
version = Version(version)
def __del__(self):
if len(self.adds) > 0 or len(self.removals) > 0:
sys.stderr.write(
"WARNING: Uncommitted config schema \"%s\" version %s\n" %
(self.name, self.current_version))
if self.current_version and self.current_version != version:
self.commit()
def version(self, version, checkpoint=False):
version = Version(version)
if checkpoint or self.data == []:
self.data.append(SchemaUpdateRecord(version, 'checkpoint'))
if self.current_version and self.current_version != version:
self.commit()
self.current_version = version
if checkpoint or self.data == []:
self.data.append(SchemaUpdateRecord(version, 'checkpoint'))
def section(self, name):
self.current_section = name
self.current_version = version
def param(self, *args, **kwargs):
self._ensure_version()
def section(self, name):
self.current_section = name
if not 'section' in kwargs and self.current_section:
kwargs['section'] = self.current_section
def param(self, *args, **kwargs):
self._ensure_version()
self.adds.append(ConfigParameterSchema(*args, **kwargs))
if not 'section' in kwargs and self.current_section:
kwargs['section'] = self.current_section
def remove_param(self, name):
self._ensure_version()
self.adds.append(ConfigParameterSchema(*args, **kwargs))
self.removals.append(name)
def remove_param(self, name):
self._ensure_version()
def commit(self):
"Finalize schema building"
self._ensure_version()
self.removals.append(name)
if len(self.removals) > 0:
self.data.append(SchemaUpdateRecord(self.current_version, 'remove', self.removals))
self.removals = []
if len(self.adds) > 0:
self.data.append(SchemaUpdateRecord(self.current_version, 'add', self.adds))
self.adds = []
def commit(self):
"Finalize schema building"
self._ensure_version()
if len(self.removals) > 0:
self.data.append(
SchemaUpdateRecord(
self.current_version,
'remove',
self.removals))
self.removals = []
if len(self.adds) > 0:
self.data.append(
SchemaUpdateRecord(
self.current_version,
'add',
self.adds))
self.adds = []
def _ensure_version(self):
if not self.current_version:
raise Error(
'Schema version is not specified. Please call version() method first')
def _ensure_version(self):
if not self.current_version:
raise Error, 'Schema version is not specified. Please call version() method first'
class ConfigSchemaRegistry:
__schemas = {}
@classmethod
def register_schema(self, project, configname=None):
if not configname:
configname = '%s.conf' % project
fullname = '%s/%s' % (project, configname)
self.__schemas[fullname] = []
return SchemaBuilder(fullname, self.__schemas[fullname])
__schemas = {}
@classmethod
def get_schema(self, project, version, configname=None):
if not configname:
configname = '%s.conf' % project
fullname = '%s/%s' % (project, configname)
version = Version(version)
@classmethod
def register_schema(self, project, configname=None):
if not configname:
configname = '%s.conf' % project
fullname = '%s/%s' % (project, configname)
self.__schemas[fullname] = []
return SchemaBuilder(fullname, self.__schemas[fullname])
if not fullname in self.__schemas:
return None
@classmethod
def get_schema(self, project, version, configname=None):
if not configname:
configname = '%s.conf' % project
fullname = '%s/%s' % (project, configname)
version = Version(version)
records = self.__schemas[fullname]
i = len(records)-1
# Find latest checkpoint prior given version
while i>=0 and not (records[i].operation == 'checkpoint' and records[i].version <= version): i-=1
if not fullname in self.__schemas:
return None
if i < 0:
return None
records = self.__schemas[fullname]
i = len(records) - 1
# Find latest checkpoint prior given version
while i >= 0 and not (records[i].operation == 'checkpoint' and records[i].version <= version):
i -= 1
parameters = []
seen_parameters = set()
last_version = None
if i < 0:
return None
while i < len(records) and records[i].version <= version:
last_version = records[i].version
if records[i].operation == 'add':
for param in records[i].data:
if param.name in seen_parameters:
old_param_index = index(parameters, lambda p: p.name == param.name)
if old_param_index != -1:
parameters[old_param_index] = param
else:
parameters.append(param)
seen_parameters.add(param.name)
elif records[i].operation == 'remove':
for param_name in records[i].data:
param_index = index(parameters, lambda p: p.name == param_name)
if index != -1:
parameters.pop(param_index)
seen_parameters.remove(param_name)
i += 1
parameters = []
seen_parameters = set()
last_version = None
return ConfigSchema(fullname, last_version, 'ini', parameters)
while i < len(records) and records[i].version <= version:
last_version = records[i].version
if records[i].operation == 'add':
for param in records[i].data:
if param.name in seen_parameters:
old_param_index = index(
parameters,
lambda p: p.name == param.name)
if old_param_index != -1:
parameters[old_param_index] = param
else:
parameters.append(param)
seen_parameters.add(param.name)
elif records[i].operation == 'remove':
for param_name in records[i].data:
param_index = index(
parameters,
lambda p: p.name == param_name)
if index != -1:
parameters.pop(param_index)
seen_parameters.remove(param_name)
i += 1
return ConfigSchema(fullname, last_version, 'ini', parameters)
class ConfigSchema:
def __init__(self, name, version, format, parameters):
self.name = name
self.version = Version(version)
self.format = format
self.parameters = parameters
def has_section(self, section):
return find(self.parameters, lambda p: p.section == section) != None
def __init__(self, name, version, format, parameters):
self.name = name
self.version = Version(version)
self.format = format
self.parameters = parameters
def has_section(self, section):
return (
find(self.parameters, lambda p: p.section == section) is not None
)
def get_parameter(self, name, section=None):
# TODO: optimize this
return (
find(
self.parameters,
lambda p: p.name == name and p.section == section)
)
def __repr__(self):
return (
'<ConfigSchema name=%s version=%s format=%s parameters=%s>' % (
self.name, self.version, self.format, self.parameters)
)
def get_parameter(self, name, section=None):
# TODO: optimize this
return find(self.parameters, lambda p: p.name == name and p.section == section)
def __repr__(self):
return '<ConfigSchema name=%s version=%s format=%s parameters=%s>' % (self.name, self.version, self.format, self.parameters)
class ConfigParameterSchema:
def __init__(self, name, type, section=None, description=None, default=None, required=False, deprecation_message=None):
self.section = section
self.name = name
self.type = type
self.description = description
self.default = default
self.required = required
self.deprecation_message = deprecation_message
def __repr__(self):
return '<ConfigParameterSchema %s>' % ' '.join(['%s=%s' % (attr, getattr(self, attr)) for attr in ['section', 'name', 'type', 'description', 'default', 'required']])
def __init__(
self,
name,
type,
section=None,
description=None,
default=None,
required=False,
deprecation_message=None):
self.section = section
self.name = name
self.type = type
self.description = description
self.default = default
self.required = required
self.deprecation_message = deprecation_message
def __repr__(self):
return (
'<ConfigParameterSchema %s>' % ' '.join(['%s=%s' % (attr, getattr(self, attr))
for attr in ['section', 'name', 'type', 'description', 'default', 'required']])
)
class TypeValidatorRegistry:
__validators = {}
@classmethod
def register_validator(self, type_name, type_validator):
self.__validators[type_name] = type_validator
__validators = {}
@classmethod
def get_validator(self, name):
return self.__validators[name]
@classmethod
def register_validator(self, type_name, type_validator):
self.__validators[type_name] = type_validator
@classmethod
def get_validator(self, name):
return self.__validators[name]
class SchemaError(Issue):
def __init__(self, message):
super(SchemaError, self).__init__(Issue.ERROR, message)
def __init__(self, message):
super(SchemaError, self).__init__(Issue.ERROR, message)
class InvalidValueError(MarkedIssue):
def __init__(self, message, mark=Mark('', 0, 0)):
super(InvalidValueError, self).__init__(Issue.ERROR, 'Invalid value: '+message, mark)
def __init__(self, message, mark=Mark('', 0, 0)):
super(
InvalidValueError,
self).__init__(
Issue.ERROR,
'Invalid value: ' +
message,
mark)
class TypeValidator(object):
def __init__(self, f):
super(TypeValidator, self).__init__()
self.f = f
def validate(self, value):
return getattr(self, 'f')(value)
def __init__(self, f):
super(TypeValidator, self).__init__()
self.f = f
def validate(self, value):
return getattr(self, 'f')(value)
def type_validator(name, **kwargs):
def wrap(fn):
def wrapped(s):
return fn(s, **kwargs)
o = TypeValidator(wrapped)
TypeValidatorRegistry.register_validator(name, o)
return fn
return wrap
def wrap(fn):
def wrapped(s):
return fn(s, **kwargs)
o = TypeValidator(wrapped)
TypeValidatorRegistry.register_validator(name, o)
return fn
return wrap
def isissue(o):
return isinstance(o, Issue)
return isinstance(o, Issue)
@type_validator('boolean')
def validate_boolean(s):
s = s.lower()
if s == 'true':
return True
elif s == 'false':
return False
else:
return InvalidValueError('Value should be "true" or "false"')
s = s.lower()
if s == 'true':
return True
elif s == 'false':
return False
else:
return InvalidValueError('Value should be "true" or "false"')
def validate_enum(s, values=[]):
if s in values:
return None
if len(values) == 0:
message = 'There should be no value'
elif len(values) == 1:
message = 'The only valid value is %s' % values[0]
else:
message = 'Valid values are %s and %s' % (', '.join(values[:-1]), values[-1])
return InvalidValueError('%s' % message)
if s in values:
return None
if len(values) == 0:
message = 'There should be no value'
elif len(values) == 1:
message = 'The only valid value is %s' % values[0]
else:
message = 'Valid values are %s and %s' % (
', '.join(values[:-1]), values[-1])
return InvalidValueError('%s' % message)
def validate_ipv4_address(s):
s = s.strip()
parts = s.split('.')
if len(parts) == 4:
if all([all([c.isdigit() for c in part]) for part in parts]):
parts = [int(part) for part in parts]
if all([part < 256 for part in parts]):
return '.'.join([str(part) for part in parts])
s = s.strip()
parts = s.split('.')
if len(parts) == 4:
if all([all([c.isdigit() for c in part]) for part in parts]):
parts = [int(part) for part in parts]
if all([part < 256 for part in parts]):
return '.'.join([str(part) for part in parts])
return InvalidValueError('Value should be ipv4 address')
return InvalidValueError('Value should be ipv4 address')
def validate_ipv4_network(s):
s = s.strip()
parts = s.split('/')
if len(parts) != 2:
return InvalidValueError('Should have "/" character separating address and prefix length')
s = s.strip()
parts = s.split('/')
if len(parts) != 2:
return (
InvalidValueError(
'Should have "/" character separating address and prefix length')
)
address, prefix = parts
prefix = prefix.strip()
address, prefix = parts
prefix = prefix.strip()
if prefix.strip() == '':
return InvalidValueError('Prefix length is required')
if prefix.strip() == '':
return InvalidValueError('Prefix length is required')
address = validate_ipv4_address(address)
if isissue(address):
return address
address = validate_ipv4_address(address)
if isissue(address):
return address
if not all([c.isdigit() for c in prefix]):
return InvalidValueError('Prefix length should be an integer')
if not all([c.isdigit() for c in prefix]):
return InvalidValueError('Prefix length should be an integer')
prefix = int(prefix)
if prefix > 32:
return InvalidValueError('Prefix length should be less than or equal to 32')
prefix = int(prefix)
if prefix > 32:
return (
InvalidValueError(
'Prefix length should be less than or equal to 32')
)
return '%s/%d' % (address, prefix)
return '%s/%d' % (address, prefix)
def validate_host_label(s):
if len(s) == 0:
return InvalidValueError('Host label should have at least one character')
if len(s) == 0:
return (
InvalidValueError('Host label should have at least one character')
)
if not s[0].isalpha():
return InvalidValueError('Host label should start with a letter, but it starts with "%s"' % s[0])
if not s[0].isalpha():
return (
InvalidValueError(
'Host label should start with a letter, but it starts with "%s"' %
s[0])
)
if len(s) == 1: return s
if len(s) == 1:
return s
if not (s[-1].isdigit() or s[-1].isalpha()):
return InvalidValueError('Host label should end with letter or digit, but it ends with "%s"' % s[-1], Mark('', 0, len(s)-1))
if not (s[-1].isdigit() or s[-1].isalpha()):
return (
InvalidValueError(
'Host label should end with letter or digit, but it ends with "%s"' %
s[-1], Mark('', 0, len(s) - 1))
)
if len(s) == 2: return s
if len(s) == 2:
return s
for i, c in enumerate(s[1:-1]):
if not (c.isalpha() or c.isdigit() or c == '-'):
return InvalidValueError('Host label should contain only letters, digits or hypens, but it contains "%s"' % c, Mark('', 0, i+1))
return s
for i, c in enumerate(s[1:-1]):
if not (c.isalpha() or c.isdigit() or c == '-'):
return (
InvalidValueError(
'Host label should contain only letters, digits or hypens, but it contains "%s"' %
c, Mark('', 0, i + 1))
)
return s
@type_validator('host')
@type_validator('host_address')
def validate_host_address(s):
result = validate_ipv4_address(s)
if not isissue(result):
return result
result = validate_ipv4_address(s)
if not isissue(result):
return result
offset = len(s) - len(s.lstrip())
offset = len(s) - len(s.lstrip())
parts = s.strip().split('.')
part_offset = offset
labels = []
for part in parts:
host_label = validate_host_label(part)
if isissue(host_label):
return host_label.offset_by(Mark('', 0, part_offset))
parts = s.strip().split('.')
part_offset = offset
labels = []
for part in parts:
host_label = validate_host_label(part)
if isissue(host_label):
return host_label.offset_by(Mark('', 0, part_offset))
part_offset += len(part)+1
labels.append(host_label)
part_offset += len(part) + 1
labels.append(host_label)
return '.'.join(labels)
return '.'.join(labels)
@type_validator('network')
@type_validator('network_address')
def validate_network_address(s):
return validate_ipv4_network(s)
return validate_ipv4_network(s)
@type_validator('host_and_port')
def validate_host_and_port(s, default_port=None):
parts = s.strip().split(':', 2)
parts = s.strip().split(':', 2)
host_address = validate_host_address(parts[0])
if isissue(host_address):
return host_address
host_address = validate_host_address(parts[0])
if isissue(host_address):
return host_address
if len(parts) == 2:
port = validate_port(parts[1])
if isissue(port):
return port
elif default_port:
port = default_port
else:
return InvalidValueError('No port specified')
if len(parts) == 2:
port = validate_port(parts[1])
if isissue(port):
return port
elif default_port:
port = default_port
else:
return InvalidValueError('No port specified')
return (host_address, port)
return (host_address, port)
@type_validator('string')
@type_validator('list')
@type_validator('multi')
def validate_string(s):
return s
return s
@type_validator('integer')
def validate_integer(s, min=None, max=None):
leading_whitespace_len = 0
while leading_whitespace_len < len(s) and s[leading_whitespace_len].isspace(): leading_whitespace_len += 1
leading_whitespace_len = 0
while leading_whitespace_len < len(s) and s[leading_whitespace_len].isspace():
leading_whitespace_len += 1
s = s.strip()
if s == '':
return InvalidValueError('Should not be empty')
s = s.strip()
if s == '':
return InvalidValueError('Should not be empty')
for i, c in enumerate(s):
if not c.isdigit() and not ((c == '-') and (i == 0)):
return InvalidValueError('Only digits are allowed, but found char "%s"' % c, Mark('', 1, i+1+leading_whitespace_len))
for i, c in enumerate(s):
if not c.isdigit() and not ((c == '-') and (i == 0)):
return (
InvalidValueError(
'Only digits are allowed, but found char "%s"' %
c, Mark('', 1, i + 1 + leading_whitespace_len))
)
v = int(s)
if min and v < min:
return InvalidValueError('Should be greater than or equal to %d' % min, Mark('', 1, leading_whitespace_len))
if max and v > max:
return InvalidValueError('Should be less than or equal to %d' % max, Mark('', 1, leading_whitespace_len))
v = int(s)
if min and v < min:
return (
InvalidValueError(
'Should be greater than or equal to %d' %
min, Mark('', 1, leading_whitespace_len))
)
if max and v > max:
return (
InvalidValueError(
'Should be less than or equal to %d' %
max, Mark('', 1, leading_whitespace_len))
)
return v
return v
@type_validator('float')
def validate_float(s):
# TODO: Implement proper validation
return float(s)
# TODO: Implement proper validation
return float(s)
@type_validator('port')
def validate_port(s, min=1, max=65535):
return validate_integer(s, min=min, max=max)
return validate_integer(s, min=min, max=max)
@type_validator('string_list')
def validate_list(s, element_type='string'):
element_type_validator = TypeValidatorRegistry.get_validator(element_type)
if not element_type_validator:
return SchemaError('Invalid element type "%s"' % element_type)
element_type_validator = TypeValidatorRegistry.get_validator(element_type)
if not element_type_validator:
return SchemaError('Invalid element type "%s"' % element_type)
result = []
s = s.strip()
result = []
s = s.strip()
if s == '':
return result
values = s.split(',')
for value in values:
validated_value = element_type_validator.validate(value.strip())
if isinstance(validated_value, Issue):
# TODO: provide better position reporting
return validated_value
result.append(validated_value)
if s == '':
return result
values = s.split(',')
for value in values:
validated_value = element_type_validator.validate(value.strip())
if isinstance(validated_value, Issue):
# TODO: provide better position reporting
return validated_value
result.append(validated_value)
return result
@type_validator('string_dict')
def validate_dict(s, element_type='string'):
element_type_validator = TypeValidatorRegistry.get_validator(element_type)
if not element_type_validator:
return SchemaError('Invalid element type "%s"' % element_type)
element_type_validator = TypeValidatorRegistry.get_validator(element_type)
if not element_type_validator:
return SchemaError('Invalid element type "%s"' % element_type)
result = {}
s = s.strip()
result = {}
s = s.strip()
if s == '':
if s == '':
return result
pairs = s.split(',')
for pair in pairs:
key_value = pair.split(':', 2)
if len(key_value) < 2:
return (
InvalidValueError(
'Value should be NAME:VALUE pairs separated by ","')
)
key, value = key_value
key = key.strip()
value = value.strip()
if key == '':
# TODO: provide better position reporting
return InvalidValueError('Key name should not be empty')
validated_value = element_type_validator.validate(value)
if isinstance(validated_value, Issue):
# TODO: provide better position reporting
return validated_value
result[key] = validated_value
return result
pairs = s.split(',')
for pair in pairs:
key_value = pair.split(':', 2)
if len(key_value) < 2:
return InvalidValueError('Value should be NAME:VALUE pairs separated by ","')
key, value = key_value
key = key.strip()
value = value.strip()
if key == '':
# TODO: provide better position reporting
return InvalidValueError('Key name should not be empty')
validated_value = element_type_validator.validate(value)
if isinstance(validated_value, Issue):
# TODO: provide better position reporting
return validated_value
result[key] = validated_value
return result

1
ostack_validator/schemas/__init__.py
View File

@ -1,4 +1,3 @@
import ostack_validator.schemas.keystone
import ostack_validator.schemas.nova
import ostack_validator.schemas.cinder

1
ostack_validator/schemas/cinder/__init__.py
View File

@ -1,2 +1 @@
import ostack_validator.schemas.cinder.v2013_1_3

3032
ostack_validator/schemas/cinder/v2013_1_3.py
View File

File diff suppressed because it is too large Load Diff

1
ostack_validator/schemas/keystone/__init__.py
View File

@ -1,2 +1 @@
import ostack_validator.schemas.keystone.v2013_1_3

901
ostack_validator/schemas/keystone/v2013_1_3.py
View File

File diff suppressed because it is too large Load Diff

1
ostack_validator/schemas/nova/__init__.py
View File

@ -1,3 +1,2 @@
import ostack_validator.schemas.nova.v2013_1
import ostack_validator.schemas.nova.v2013_1_3

1334
ostack_validator/schemas/nova/v2013_1.py
View File

File diff suppressed because it is too large Load Diff

4962
ostack_validator/schemas/nova/v2013_1_3.py
View File

File diff suppressed because it is too large Load Diff

3
ostack_validator/schemas/schema_generator.py
View File

@ -4,6 +4,7 @@ import sys
class SchemaParser(object):
def parse_args(self, argv):
parser = argparse.ArgumentParser()
parser.add_argument('--conf', dest='conf_file', default=None,
@ -79,4 +80,4 @@ class SchemaParser(object):
if __name__ == '__main__':
runner = SchemaParser()
runner.run(sys.argv)
runner.run(sys.argv)

69
ostack_validator/test_config_schema_registry.py
View File

@ -3,46 +3,57 @@ from ostack_validator.common import find
import unittest
class ConfigSchemaRegistryTests(unittest.TestCase):
def test_sample(self):
nova = ConfigSchemaRegistry.register_schema(project='nova')
nova.version('1.0.0')
nova.section('DEFAULT')
nova.param(name='verbose', type='boolean')
nova.param(name='rabbit_host', type='address')
def test_sample(self):
nova = ConfigSchemaRegistry.register_schema(project='nova')
nova.version('1.1.0')
nova.section('DEFAULT')
nova.param(name='verbose', type='boolean', default=False)
nova.remove_param('rabbit_host')
nova.version('1.0.0')
nova.section('DEFAULT')
nova.param(name='verbose', type='boolean')
nova.param(name='rabbit_host', type='address')
nova.commit()
nova.version('1.1.0')
nova.section('DEFAULT')
nova.param(name='verbose', type='boolean', default=False)
nova.remove_param('rabbit_host')
schema10 = ConfigSchemaRegistry.get_schema(project='nova', version='1.0.0')
nova.commit()
self.assertEqual(Version('1.0.0'), schema10.version)
self.assertEqual('ini', schema10.format)
schema10 = ConfigSchemaRegistry.get_schema(
project='nova', version='1.0.0')
verbose_param = find(schema10.parameters, lambda p: p.name == 'verbose')
self.assertIsNotNone(verbose_param)
self.assertEqual('boolean', verbose_param.type)
self.assertEqual(None, verbose_param.default)
self.assertEqual(Version('1.0.0'), schema10.version)
self.assertEqual('ini', schema10.format)
rabbit_host_param = find(schema10.parameters, lambda p: p.name == 'rabbit_host')
self.assertIsNotNone(rabbit_host_param)
self.assertEqual('address', rabbit_host_param.type)
verbose_param = find(
schema10.parameters,
lambda p: p.name == 'verbose')
self.assertIsNotNone(verbose_param)
self.assertEqual('boolean', verbose_param.type)
self.assertEqual(None, verbose_param.default)
schema11 = ConfigSchemaRegistry.get_schema(project='nova', version='1.1.0')
rabbit_host_param = find(
schema10.parameters,
lambda p: p.name == 'rabbit_host')
self.assertIsNotNone(rabbit_host_param)
self.assertEqual('address', rabbit_host_param.type)
verbose_param11 = find(schema11.parameters, lambda p: p.name == 'verbose')
self.assertIsNotNone(verbose_param11)
self.assertEqual(False, verbose_param11.default)
schema11 = ConfigSchemaRegistry.get_schema(
project='nova', version='1.1.0')
rabbit_host_param11 = find(schema11.parameters, lambda p: p.name == 'rabbit_host')
self.assertIsNone(rabbit_host_param11)
verbose_param11 = find(
schema11.parameters,
lambda p: p.name == 'verbose')
self.assertIsNotNone(verbose_param11)
self.assertEqual(False, verbose_param11.default)
rabbit_host_param11 = find(
schema11.parameters,
lambda p: p.name == 'rabbit_host')
self.assertIsNone(rabbit_host_param11)
if __name__ == '__main__':
unittest.main()
unittest.main()

284
ostack_validator/test_configuration.py
View File

@ -2,204 +2,210 @@ import unittest
from ostack_validator.config_model import Configuration
class ConfigurationTests(unittest.TestCase):
section = 'section1'
param = 'param1'
fullparam = '%s.%s' % (section, param)
value = 'foobar'
default_value = 'bar123'
section = 'section1'
param = 'param1'
fullparam = '%s.%s' % (section, param)
value = 'foobar'
default_value = 'bar123'
def test_empty(self):
c = Configuration()
self.assertIsNone(c.get('section1.param1'))
def test_storage(self):
c = Configuration()
c.set(self.fullparam, self.value)
def test_empty(self):
c = Configuration()
self.assertIsNone(c.get('section1.param1'))
self.assertEqual(self.value, c.get(self.fullparam))
def test_storage(self):
c = Configuration()
c.set(self.fullparam, self.value)
def test_parameter_names_containing_sections(self):
c = Configuration()
c.set(self.fullparam, self.value)
self.assertEqual(self.value, c.get(self.fullparam))
self.assertEqual(self.value, c.get('%s.%s' % (self.section, self.param)))
def test_parameter_with_default_section(self):
c = Configuration()
c.set(self.param, self.value)
def test_parameter_names_containing_sections(self):
c = Configuration()
c.set(self.fullparam, self.value)
self.assertEqual(self.value, c.get(self.param))
self.assertEqual(
self.value, c.get('%s.%s' %
(self.section, self.param)))
def test_explicit_default_on_get(self):
c = Configuration()
override_value = '12345'
def test_parameter_with_default_section(self):
c = Configuration()
c.set(self.param, self.value)
self.assertEqual(override_value, c.get(self.fullparam, default=override_value))
self.assertEqual(self.value, c.get(self.param))
def test_default(self):
c = Configuration()
c.set_default(self.fullparam, self.default_value)
def test_explicit_default_on_get(self):
c = Configuration()
override_value = '12345'
self.assertEqual(self.default_value, c.get(self.fullparam))
self.assertEqual(
override_value,
c.get(self.fullparam,
default=override_value))
def test_normal_overrides_default(self):
c = Configuration()
c.set(self.fullparam, self.value)
c.set_default(self.fullparam, self.default_value)
def test_default(self):
c = Configuration()
c.set_default(self.fullparam, self.default_value)
self.assertEqual(self.value, c.get(self.fullparam))
self.assertEqual(self.default_value, c.get(self.fullparam))
def test_contains(self):
c = Configuration()
self.assertFalse(c.contains(self.fullparam))
def test_normal_overrides_default(self):
c = Configuration()
c.set(self.fullparam, self.value)
c.set_default(self.fullparam, self.default_value)
def test_contains_default(self):
c = Configuration()
c.set_default(self.fullparam, self.default_value)
self.assertEqual(self.value, c.get(self.fullparam))
self.assertTrue(c.contains(self.fullparam))
self.assertFalse(c.contains(self.fullparam, ignoreDefault=True))
def test_contains(self):
c = Configuration()
self.assertFalse(c.contains(self.fullparam))
def test_contains_normal(self):
c = Configuration()
c.set(self.fullparam, self.value)
def test_contains_default(self):
c = Configuration()
c.set_default(self.fullparam, self.default_value)
self.assertTrue(c.contains(self.fullparam))
self.assertTrue(c.contains(self.fullparam, ignoreDefault=True))
self.assertTrue(c.contains(self.fullparam))
self.assertFalse(c.contains(self.fullparam, ignoreDefault=True))
def test_contains_normal(self):
c = Configuration()
c.set(self.fullparam, self.value)
def test_is_default_returns_false_if_param_missing(self):
c = Configuration()
self.assertFalse(c.is_default(self.fullparam))
self.assertTrue(c.contains(self.fullparam))
self.assertTrue(c.contains(self.fullparam, ignoreDefault=True))
def test_is_default_returns_true_if_only_default_value_set(self):
c = Configuration()
c.set_default(self.fullparam, self.default_value)
def test_is_default_returns_false_if_param_missing(self):
c = Configuration()
self.assertFalse(c.is_default(self.fullparam))
self.assertTrue(c.is_default(self.fullparam))
def test_is_default_returns_true_if_only_default_value_set(self):
c = Configuration()
c.set_default(self.fullparam, self.default_value)
def test_is_default_returns_false_if_normal_value_set(self):
c = Configuration()
c.set(self.fullparam, self.value)
self.assertTrue(c.is_default(self.fullparam))
self.assertFalse(c.is_default(self.fullparam))
def test_is_default_returns_false_if_normal_value_set(self):
c = Configuration()
c.set(self.fullparam, self.value)
def test_is_default_returns_false_if_both_values_set(self):
c = Configuration()
c.set_default(self.fullparam, self.default_value)
c.set(self.fullparam, self.value)
self.assertFalse(c.is_default(self.fullparam))
self.assertFalse(c.is_default(self.fullparam))
def test_is_default_returns_false_if_both_values_set(self):
c = Configuration()
c.set_default(self.fullparam, self.default_value)
c.set(self.fullparam, self.value)
def test_subsection_set(self):
c = Configuration()
c.section(self.section).set(self.param, self.value)
self.assertFalse(c.is_default(self.fullparam))
self.assertEqual(self.value, c.get(self.fullparam))
def test_subsection_set(self):
c = Configuration()
c.section(self.section).set(self.param, self.value)
def test_keys(self):
c = Configuration()
c.set_default('section1.param1', '123')
c.set('section2.param1', '456')
self.assertEqual(self.value, c.get(self.fullparam))
self.assertEqual(['section1', 'section2'], sorted(c.keys()))
def test_keys(self):
c = Configuration()
c.set_default('section1.param1', '123')
c.set('section2.param1', '456')
def test_subsection_keys(self):
c = Configuration()
c.set_default('%s.param1' % self.section, '123')
c.set('%s.param2' % self.section, '456')
self.assertEqual(['section1', 'section2'], sorted(c.keys()))
self.assertEqual(['param1', 'param2'], sorted(c.section(self.section).keys()))
def test_subsection_keys(self):
c = Configuration()
c.set_default('%s.param1' % self.section, '123')
c.set('%s.param2' % self.section, '456')
def test_subsection_items(self):
c = Configuration()
c.set('%s.param1' % self.section, 'value1')
c.set_default('%s.param2' % self.section, 'value2')
self.assertEqual(
['param1', 'param2'], sorted(c.section(self.section).keys()))
self.assertEqual([('param1', 'value1'), ('param2', 'value2')], sorted(c.section(self.section).items()))
def test_subsection_items(self):
c = Configuration()
c.set('%s.param1' % self.section, 'value1')
c.set_default('%s.param2' % self.section, 'value2')
def test_subsection_get(self):
c = Configuration()
self.assertEqual(
[('param1', 'value1'), ('param2', 'value2')], sorted(c.section(self.section).items()))
c.set(self.fullparam, self.value)
def test_subsection_get(self):
c = Configuration()
cs = c.section(self.section)
self.assertEqual(self.value, cs.get(self.param))
c.set(self.fullparam, self.value)
def test_getitem(self):
c = Configuration()
c.set(self.fullparam, self.value)
cs = c.section(self.section)
self.assertEqual(self.value, cs.get(self.param))
self.assertEqual(self.value, c[self.fullparam])
def test_getitem(self):
c = Configuration()
c.set(self.fullparam, self.value)
def test_subsection_getitem(self):
c = Configuration()
c.set(self.fullparam, self.value)
self.assertEqual(self.value, c[self.fullparam])
cs = c.section(self.section)
def test_subsection_getitem(self):
c = Configuration()
c.set(self.fullparam, self.value)
self.assertEqual(self.value, cs[self.param])
cs = c.section(self.section)
def test_setitem(self):
c = Configuration()
self.assertEqual(self.value, cs[self.param])
c[self.fullparam] = self.value
def test_setitem(self):
c = Configuration()
self.assertEqual(self.value, c.get(self.fullparam))
c[self.fullparam] = self.value
def test_subsection_setitem(self):
c = Configuration()
self.assertEqual(self.value, c.get(self.fullparam))
cs = c.section(self.section)
def test_subsection_setitem(self):
c = Configuration()
cs[self.param] = self.value
cs = c.section(self.section)
self.assertEqual(self.value, c.get(self.fullparam))
cs[self.param] = self.value
def test_contains(self):
c = Configuration()
self.assertEqual(self.value, c.get(self.fullparam))
self.assertFalse(self.section in c)
def test_contains(self):
c = Configuration()
c.set(self.fullparam, self.value)
self.assertTrue(self.section in c)
self.assertFalse(self.section in c)
def test_subsection_contains(self):
c = Configuration()
c.set(self.fullparam, self.value)
self.assertTrue(self.section in c)
c.set('section1.param1', '123')
c.set_default('section2.param2', '234')
def test_subsection_contains(self):
c = Configuration()
self.assertTrue('param1' in c.section('section1'))
self.assertTrue('param2' in c.section('section2'))
self.assertFalse('param1' in c.section('section2'))
def test_returns_section_object_even_if_section_doesnot_exist(self):
c = Configuration()
self.assertIsNotNone(c.section('foo'))
c.set('section1.param1', '123')
c.set_default('section2.param2', '234')
def test_template_substitution(self):
c = Configuration()
c.set('a', 'x')
c.set('b', '$a')
c.set('c', '$b')
self.assertTrue('param1' in c.section('section1'))
self.assertTrue('param2' in c.section('section2'))
self.assertFalse('param1' in c.section('section2'))
self.assertEqual('x', c.get('c'))
def test_returns_section_object_even_if_section_doesnot_exist(self):
c = Configuration()
self.assertIsNotNone(c.section('foo'))
def test_cycle_template_substitution_resolves_in_empty_string(self):
c = Configuration()
c.set('a', 'a$c')
c.set('b', 'b$a')
c.set('c', 'c$b')
def test_template_substitution(self):
c = Configuration()
c.set('a', 'x')
c.set('b', '$a')
c.set('c', '$b')
self.assertEqual('cba', c.get('c'))
self.assertEqual('x', c.get('c'))
def test_getting_raw_values(self):
c = Configuration()
def test_cycle_template_substitution_resolves_in_empty_string(self):
c = Configuration()
c.set('a', 'a$c')
c.set('b', 'b$a')
c.set('c', 'c$b')
c.set('a', '$b')
c.set('b', 'x')
self.assertEqual('cba', c.get('c'))
self.assertEqual('$b', c.get('a', raw=True))
def test_getting_raw_values(self):
c = Configuration()
c.set('a', '$b')
c.set('b', 'x')
self.assertEqual('$b', c.get('a', raw=True))

25
ostack_validator/test_mark.py
View File

@ -2,20 +2,21 @@ from ostack_validator.common import Mark
import unittest
class MarkTests(unittest.TestCase):
def test_creation(self):
m = Mark('nova.conf', 3, 5)
self.assertEqual('nova.conf', m.source)
self.assertEqual(3, m.line)
self.assertEqual(5, m.column)
def test_merge(self):
m1 = Mark('nova.conf', 3, 5)
m2 = Mark('unknown', 2, 7)
def test_creation(self):
m = Mark('nova.conf', 3, 5)
self.assertEqual('nova.conf', m.source)
self.assertEqual(3, m.line)
self.assertEqual(5, m.column)
m = m1.merge(m2)
def test_merge(self):
m1 = Mark('nova.conf', 3, 5)
m2 = Mark('unknown', 2, 7)
self.assertEqual(m1.source, m.source)
self.assertEqual(m1.line + m2.line, m.line)
self.assertEqual(m1.column + m2.column, m.column)
m = m1.merge(m2)
self.assertEqual(m1.source, m.source)
self.assertEqual(m1.line + m2.line, m.line)
self.assertEqual(m1.column + m2.column, m.column)

361
ostack_validator/test_type_validators.py
View File

@ -3,265 +3,276 @@ from ostack_validator.schema import TypeValidatorRegistry
import unittest
class TypeValidatorTestHelper(object):
def setUp(self):
super(TypeValidatorTestHelper, self).setUp()
self.validator = TypeValidatorRegistry.get_validator(self.type_name)
def assertValid(self, value):
self.assertNotIsInstance(self.validator.validate(value), Issue)
def setUp(self):
super(TypeValidatorTestHelper, self).setUp()
self.validator = TypeValidatorRegistry.get_validator(self.type_name)
def assertValid(self, value):
self.assertNotIsInstance(self.validator.validate(value), Issue)
def assertInvalid(self, value):
self.assertIsInstance(self.validator.validate(value), Issue)
def assertInvalid(self, value):
self.assertIsInstance(self.validator.validate(value), Issue)
class StringTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'string'
type_name = 'string'
def test_empty_string_passes(self):
self.assertValid('')
def test_empty_string_passes(self):
self.assertValid('')
def test_validation_always_passes(self):
self.assertValid('foo bar')
def test_validation_always_passes(self):
self.assertValid('foo bar')
def test_should_return_same_string_if_valid(self):
s = 'foo bar'
self.assertEqual(s, self.validator.validate(s))
def test_should_return_same_string_if_valid(self):
s = 'foo bar'
self.assertEqual(s, self.validator.validate(s))
class BooleanTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'boolean'
type_name = 'boolean'
def test_True(self):
v = self.validator.validate('True')
self.assertEqual(True, v)
def test_True(self):
v = self.validator.validate('True')
self.assertEqual(True, v)
def test_False(self):
v = self.validator.validate('False')
self.assertEqual(False, v)
def test_False(self):
v = self.validator.validate('False')
self.assertEqual(False, v)
def test_other_values_produce_error(self):
self.assertInvalid('foo')
def test_other_values_produce_error(self):
self.assertInvalid('foo')
class IntegerTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'integer'
type_name = 'integer'
def test_positive_values_are_valid(self):
self.assertValid('123')
def test_positive_values_are_valid(self):
self.assertValid('123')
def test_zero_is_valid(self):
self.assertValid('0')
def test_zero_is_valid(self):
self.assertValid('0')
def test_negative_values_are_valid(self):
self.assertValid('-123')
def test_negative_values_are_valid(self):
self.assertValid('-123')
def test_leading_whitespace_is_ignored(self):
self.assertValid(' 5')
def test_leading_whitespace_is_ignored(self):
self.assertValid(' 5')
def test_trailing_whitespace_is_ignored(self):
self.assertValid('7 ')
def test_trailing_whitespace_is_ignored(self):
self.assertValid('7 ')
def test_non_digits_are_invalid(self):
self.assertInvalid('12a45')
def test_non_digits_are_invalid(self):
self.assertInvalid('12a45')
def test_invalid_char_error_contains_proper_column_in_mark(self):
error = self.validator.validate('12a45')
self.assertIsInstance(error, MarkedIssue)
self.assertEqual(3, error.mark.column)
def test_invalid_char_error_contains_proper_column_in_mark(self):
error = self.validator.validate('12a45')
self.assertIsInstance(error, MarkedIssue)
self.assertEqual(3, error.mark.column)
def test_invalid_char_error_contains_proper_column_if_leading_whitespaces(self):
error = self.validator.validate(' 12a45')
self.assertIsInstance(error, MarkedIssue)
self.assertEqual(5, error.mark.column)
def test_invalid_char_error_contains_proper_column_if_leading_whitespaces(
self):
error = self.validator.validate(' 12a45')
self.assertIsInstance(error, MarkedIssue)
self.assertEqual(5, error.mark.column)
def test_returns_integer_if_valid(self):
v = self.validator.validate('123')
self.assertEqual(123, v)
def test_returns_integer_if_valid(self):
v = self.validator.validate('123')
self.assertEqual(123, v)
class HostAddressTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'host_address'
type_name = 'host_address'
def test_ipv4_address(self):
self.assertValid('127.0.0.1')
def test_ipv4_address(self):
self.assertValid('127.0.0.1')
def test_returns_address(self):
s = '10.0.0.1'
v = self.validator.validate(s)
self.assertEqual(s, v)
def test_returns_address(self):
s = '10.0.0.1'
v = self.validator.validate(s)
self.assertEqual(s, v)
def test_value_with_less_than_4_numbers_separated_by_dots(self):
self.assertInvalid('10.0.0')
def test_value_with_less_than_4_numbers_separated_by_dots(self):
self.assertInvalid('10.0.0')
def test_ipv4_like_string_with_numbers_greater_than_255(self):
self.assertInvalid('10.0.256.1')
def test_ipv4_like_string_with_numbers_greater_than_255(self):
self.assertInvalid('10.0.256.1')
def test_host_name(self):
self.assertValid('foo.bar.baz')
def test_host_name(self):
self.assertValid('foo.bar.baz')
def test_host_with_empty_parts(self):
self.assertInvalid('.foo.bar')
self.assertInvalid('foo..bar')
self.assertInvalid('foo.bar.')
def test_host_with_empty_parts(self):
self.assertInvalid('.foo.bar')
self.assertInvalid('foo..bar')
self.assertInvalid('foo.bar.')
def test_host_parts_with_invalid_chars(self):
self.assertInvalid('foo.ba r.baz')
self.assertInvalid('foo.x_y.bar')
def test_host_parts_with_invalid_chars(self):
self.assertInvalid('foo.ba r.baz')
self.assertInvalid('foo.x_y.bar')
def test_host_with_single_host_label(self):
self.assertValid('foo')
def test_host_with_single_host_label(self):
self.assertValid('foo')
def test_host_part_starting_with_non_letter(self):
self.assertInvalid('123foo')
def test_host_part_starting_with_non_letter(self):
self.assertInvalid('123foo')
def test_host_that_ends_with_a_hyphen(self):
self.assertInvalid('foo-')
def test_host_that_ends_with_a_hyphen(self):
self.assertInvalid('foo-')
def test_mark_should_point_to_incorrect_symbol(self):
e = self.validator.validate('')
self.assertEqual(0, e.mark.column)
def test_mark_should_point_to_incorrect_symbol(self):
e = self.validator.validate('')
self.assertEqual(0, e.mark.column)
e = self.validator.validate('123foo')
self.assertEqual(0, e.mark.column)
e = self.validator.validate('123foo')
self.assertEqual(0, e.mark.column)
e = self.validator.validate('foo-')
self.assertEqual(3, e.mark.column)
e = self.validator.validate('foo-')
self.assertEqual(3, e.mark.column)
e = self.validator.validate('foo.bar.-baz')
self.assertEqual(8, e.mark.column)
e = self.validator.validate('foo.bar.-baz')
self.assertEqual(8, e.mark.column)
class NetworkAddressTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'network_address'
type_name = 'network_address'
def test_ipv4_network(self):
self.assertValid('127.0.0.1/24')
def test_ipv4_network(self):
self.assertValid('127.0.0.1/24')
def test_returns_address(self):
s = '10.0.0.1/32'
v = self.validator.validate(s)
self.assertEqual(s, v)
def test_returns_address(self):
s = '10.0.0.1/32'
v = self.validator.validate(s)
self.assertEqual(s, v)
def test_value_with_less_than_4_numbers_separated_by_dots(self):
self.assertInvalid('10.0.0/24')
def test_value_with_less_than_4_numbers_separated_by_dots(self):
self.assertInvalid('10.0.0/24')
def test_ipv4_like_string_with_numbers_greater_than_255(self):
self.assertInvalid('10.0.256.1/24')
def test_ipv4_like_string_with_numbers_greater_than_255(self):
self.assertInvalid('10.0.256.1/24')
def test_no_prefix_length(self):
self.assertInvalid('10.0.0.0')
self.assertInvalid('10.0.0.0/')
def test_no_prefix_length(self):
self.assertInvalid('10.0.0.0')
self.assertInvalid('10.0.0.0/')
def test_non_integer_prefix_length(self):
self.assertInvalid('10.0.0.0/1a')
def test_non_integer_prefix_length(self):
self.assertInvalid('10.0.0.0/1a')
def test_prefix_greater_than_32(self):
self.assertInvalid('10.0.0.0/33')
def test_prefix_greater_than_32(self):
self.assertInvalid('10.0.0.0/33')
class PortTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'port'
type_name = 'port'
def test_empty(self):
self.assertInvalid('')
def test_empty(self):
self.assertInvalid('')
def test_positive_integer(self):
self.assertValid('123')
def test_positive_integer(self):
self.assertValid('123')
def test_zero_invalid(self):
self.assertInvalid('0')
def test_zero_invalid(self):
self.assertInvalid('0')
def test_negatives_are_invalid(self):
self.assertInvalid('-1')
def test_negatives_are_invalid(self):
self.assertInvalid('-1')
def test_values_greater_than_65535_are_invalid(self):
self.assertInvalid('65536')
def test_values_greater_than_65535_are_invalid(self):
self.assertInvalid('65536')
def test_low_boundary_is_valid(self):
self.assertValid('1')
def test_low_boundary_is_valid(self):
self.assertValid('1')
def test_high_boundary_is_valid(self):
self.assertValid('65535')
def test_high_boundary_is_valid(self):
self.assertValid('65535')
def test_non_digits_are_invalid(self):
self.assertInvalid('12a5')
def test_non_digits_are_invalid(self):
self.assertInvalid('12a5')
def test_leading_and_or_trailing_whitespace_is_ignored(self):
self.assertValid(' 123')
self.assertValid('456 ')
self.assertValid(' 123 ')
def test_leading_and_or_trailing_whitespace_is_ignored(self):
self.assertValid(' 123')
self.assertValid('456 ')
self.assertValid(' 123 ')
def test_returns_integer_if_valid(self):
v = self.validator.validate('123')
self.assertEqual(123, v)
def test_returns_integer_if_valid(self):
v = self.validator.validate('123')
self.assertEqual(123, v)
class HostAndPortTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'host_and_port'
type_name = 'host_and_port'
def test_ipv4_address(self):
self.assertValid('127.0.0.1:80')
def test_ipv4_address(self):
self.assertValid('127.0.0.1:80')
def test_returns_address(self):
s = '10.0.0.1:80'
v = self.validator.validate(s)
self.assertEqual(('10.0.0.1', 80), v)
def test_returns_address(self):
s = '10.0.0.1:80'
v = self.validator.validate(s)
self.assertEqual(('10.0.0.1', 80), v)
def test_value_with_less_than_4_numbers_separated_by_dots(self):
self.assertInvalid('10.0.0:1234')
def test_value_with_less_than_4_numbers_separated_by_dots(self):
self.assertInvalid('10.0.0:1234')
def test_ipv4_like_string_with_numbers_greater_than_255(self):
self.assertInvalid('10.0.256.1:1234')
def test_ipv4_like_string_with_numbers_greater_than_255(self):
self.assertInvalid('10.0.256.1:1234')
def test_no_port(self):
self.assertInvalid('10.0.0.1')
self.assertInvalid('10.0.0.1:')
def test_no_port(self):
self.assertInvalid('10.0.0.1')
self.assertInvalid('10.0.0.1:')
def test_port_is_not_an_integer(self):
self.assertInvalid('10.0.0.1:abc')
def test_port_is_not_an_integer(self):
self.assertInvalid('10.0.0.1:abc')
def test_port_is_greater_than_65535(self):
self.assertInvalid('10.0.0.1:65536')
def test_port_is_greater_than_65535(self):
self.assertInvalid('10.0.0.1:65536')
class StringListTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'string_list'
type_name = 'string_list'
def test_empty_value(self):
v = self.validator.validate('')
self.assertEqual([], v)
def test_empty_value(self):
v = self.validator.validate('')
self.assertEqual([], v)
def test_single_value(self):
v = self.validator.validate(' foo bar ')
def test_single_value(self):
v = self.validator.validate(' foo bar ')
self.assertIsInstance(v, list)
self.assertEqual('foo bar', v[0])
self.assertEqual(1, len(v))
self.assertIsInstance(v, list)
self.assertEqual('foo bar', v[0])
self.assertEqual(1, len(v))
def test_list_of_values(self):
v = self.validator.validate(' foo bar, baz ')
def test_list_of_values(self):
v = self.validator.validate(' foo bar, baz ')
self.assertIsInstance(v, list)
self.assertEqual('foo bar', v[0])
self.assertEqual('baz', v[1])
self.assertEqual(2, len(v))
self.assertIsInstance(v, list)
self.assertEqual('foo bar', v[0])
self.assertEqual('baz', v[1])
self.assertEqual(2, len(v))
class StringDictTypeValidatorTests(TypeValidatorTestHelper, unittest.TestCase):
type_name = 'string_dict'
type_name = 'string_dict'
def test_empty_value(self):
v = self.validator.validate('')
self.assertEqual({}, v)
def test_empty_value(self):
v = self.validator.validate('')
self.assertEqual({}, v)
def test_single_value(self):
v = self.validator.validate(' foo: bar ')
def test_single_value(self):
v = self.validator.validate(' foo: bar ')
self.assertIsInstance(v, dict)
self.assertEqual('bar', v['foo'])
self.assertEqual(1, len(v))
self.assertIsInstance(v, dict)
self.assertEqual('bar', v['foo'])
self.assertEqual(1, len(v))
def test_list_of_values(self):
v = self.validator.validate(' foo: bar, baz: 123 ')
def test_list_of_values(self):
v = self.validator.validate(' foo: bar, baz: 123 ')
self.assertIsInstance(v, dict)
self.assertEqual('bar', v['foo'])
self.assertEqual('123', v['baz'])
self.assertEqual(2, len(v))
self.assertIsInstance(v, dict)
self.assertEqual('bar', v['foo'])
self.assertEqual('123', v['baz'])
self.assertEqual(2, len(v))
if __name__ == '__main__':
unittest.main()
unittest.main()

88
ostack_validator/test_version.py
View File

@ -2,62 +2,62 @@ from ostack_validator.schema import Version
import unittest
class VersionTests(unittest.TestCase):
def test_creation_from_components(self):
v = Version(1, 3, 7)
self.assertEqual(1, v.major)
self.assertEqual(3, v.minor)
self.assertEqual(7, v.maintenance)
def test_creation_from_components(self):
v = Version(1, 3, 7)
self.assertEqual(1, v.major)
self.assertEqual(3, v.minor)
self.assertEqual(7, v.maintenance)
def test_creation_from_string(self):
v = Version('1.2.12')
self.assertEqual(1, v.major)
self.assertEqual(2, v.minor)
self.assertEqual(12, v.maintenance)
def test_creation_from_string(self):
v = Version('1.2.12')
self.assertEqual(1, v.major)
self.assertEqual(2, v.minor)
self.assertEqual(12, v.maintenance)
def test_creation_from_string_with_less_parts(self):
v = Version('1.2')
self.assertEqual(1, v.major)
self.assertEqual(2, v.minor)
self.assertEqual(0, v.maintenance)
def test_creation_from_string_with_less_parts(self):
v = Version('1.2')
self.assertEqual(1, v.major)
self.assertEqual(2, v.minor)
self.assertEqual(0, v.maintenance)
v = Version('12')
self.assertEqual(12, v.major)
self.assertEqual(0, v.minor)
self.assertEqual(0, v.maintenance)
v = Version('12')
self.assertEqual(12, v.major)
self.assertEqual(0, v.minor)
self.assertEqual(0, v.maintenance)
def test_creation_from_other_version(self):
v = Version('1.2.3')
v2 = Version(v)
self.assertEqual(1, v2.major)
self.assertEqual(2, v2.minor)
self.assertEqual(3, v2.maintenance)
def test_creation_from_other_version(self):
v = Version('1.2.3')
v2 = Version(v)
self.assertEqual(1, v2.major)
self.assertEqual(2, v2.minor)
self.assertEqual(3, v2.maintenance)
def test_equility(self):
v1 = Version('1.2.3')
v2 = Version(1, 2, 3)
v3 = Version(1, 2, 4)
def test_equility(self):
v1 = Version('1.2.3')
v2 = Version(1, 2, 3)
v3 = Version(1, 2, 4)
self.assertTrue(v1 == v2)
self.assertFalse(v1 == v3)
self.assertTrue(v1 == v2)
self.assertFalse(v1 == v3)
def test_non_equility(self):
v1 = Version('1.2.3')
v2 = Version(1, 2, 3)
v3 = Version(1, 2, 4)
def test_non_equility(self):
v1 = Version('1.2.3')
v2 = Version(1, 2, 3)
v3 = Version(1, 2, 4)
self.assertFalse(v1 != v2)
self.assertTrue(v1 != v3)
self.assertFalse(v1 != v2)
self.assertTrue(v1 != v3)
def test_comparision(self):
v1 = Version('1.2.3')
v2 = Version(1, 1, 5)
def test_comparision(self):
v1 = Version('1.2.3')
v2 = Version(1, 1, 5)
self.assertTrue(v1 > v2)
self.assertFalse(v1 < v2)
self.assertTrue(v1 > v2)
self.assertFalse(v1 < v2)
if __name__ == '__main__':
unittest.main()
unittest.main()

55
ostack_validator/utils.py
View File

@ -1,29 +1,34 @@
import collections
import functools
class memoized(object):
'''Decorator. Caches a function's return value each time it is called.
If called later with the same arguments, the cached value is returned
(not reevaluated).
'''
def __init__(self, func):
self.func = func
self.cache = {}
def __call__(self, *args):
if not isinstance(args, collections.Hashable):
# uncacheable. a list, for instance.
# better to not cache than blow up.
return self.func(*args)
if args in self.cache:
return self.cache[args]
else:
value = self.func(*args)
self.cache[args] = value
return value
def __repr__(self):
'''Return the function's docstring.'''
return self.func.__doc__
def __get__(self, obj, objtype):
'''Support instance methods.'''
return functools.partial(self.__call__, obj)
class memoized(object):
'''Decorator. Caches a function's return value each time it is called.
If called later with the same arguments, the cached value is returned
(not reevaluated).
'''
def __init__(self, func):
self.func = func
self.cache = {}
def __call__(self, *args):
if not isinstance(args, collections.Hashable):
# uncacheable. a list, for instance.
# better to not cache than blow up.
return self.func(*args)
if args in self.cache:
return self.cache[args]
else:
value = self.func(*args)
self.cache[args] = value
return value
def __repr__(self):
'''Return the function's docstring.'''
return self.func.__doc__
def __get__(self, obj, objtype):
'''Support instance methods.'''
return functools.partial(self.__call__, obj)

94
ostack_validator/webui.py
View File

@ -14,67 +14,91 @@ app = Flask(__name__)
Bootstrap(app)
app.debug = True
app.config.update(
WTF_CSRF_SECRET_KEY = 'foo bar baz'
WTF_CSRF_SECRET_KEY='foo bar baz'
)
app.secret_key = 'A0Zr98j/3fooN]LWX/,?RT'
class ValidationLaunchForm(Form):
nodes = StringField('Nodes', validators=[DataRequired()])
username = StringField('Username', default='root', validators=[DataRequired()])
private_key = TextAreaField('Private Key', validators=[DataRequired()])
nodes = StringField('Nodes', validators=[DataRequired()])
username = StringField(
'Username',
default='root',
validators=[DataRequired()])
private_key = TextAreaField('Private Key', validators=[DataRequired()])
launch = SubmitField('Launch validation')
launch = SubmitField('Launch validation')
@app.template_filter()
def to_label(s):
if s in [Issue.FATAL, Issue.ERROR]:
return 'label-danger'
elif s == Issue.WARNING:
return 'label-warning'
else:
return 'label-info'
if s in [Issue.FATAL, Issue.ERROR]:
return 'label-danger'
elif s == Issue.WARNING:
return 'label-warning'
else:
return 'label-info'
@app.route('/')
def index():
return redirect('/validation')
return redirect('/validation')
@app.route('/validation', methods=['GET', 'POST'])
def launch_validation():
form = ValidationLaunchForm()
if form.validate_on_submit():
request = InspectionRequest(form.nodes.data.split(' '), form.username.data, private_key=form.private_key.data)
form = ValidationLaunchForm()
if form.validate_on_submit():
request = InspectionRequest(
form.nodes.data.split(
' '),
form.username.data,
private_key=form.private_key.data)
job = ostack_inspect_task.delay(request)
job = ostack_inspect_task.delay(request)
return redirect('/validation/%s' % job.id)
else:
return render_template('validation_form.html', form=form)
return redirect('/validation/%s' % job.id)
else:
return render_template('validation_form.html', form=form)
@app.route('/validation/<id>')
def job(id):
job = celery.AsyncResult(id)
if job.ready():
r = job.result.request
job = celery.AsyncResult(id)
if job.ready():
r = job.result.request
form = ValidationLaunchForm()
form.nodes.data = ' '.join(r.nodes)
form.username.data = r.username
form.private_key.data = r.private_key
form = ValidationLaunchForm()
form.nodes.data = ' '.join(r.nodes)
form.username.data = r.username
form.private_key.data = r.private_key
openstack = job.result.value
openstack = job.result.value
if isinstance(openstack, Openstack):
issue_source_f = lambda i: i.mark.source if isinstance(i, MarkedIssue) else None
source_groupped_issues = groupby(sorted(openstack.issues, key=issue_source_f), key=issue_source_f)
if isinstance(openstack, Openstack):
issue_source_f = lambda i: i.mark.source if isinstance(
i, MarkedIssue) else None
source_groupped_issues = groupby(
sorted(openstack.issues,
key=issue_source_f),
key=issue_source_f)
return render_template('validation_result.html', form=form, openstack=openstack, grouped_issues=source_groupped_issues)
return (
render_template(
'validation_result.html',
form=form,
openstack=openstack,
grouped_issues=source_groupped_issues)
)
else:
return (
render_template(
'validation_error.html',
form=form,
message=openstack)
)
else:
return render_template('validation_error.html', form=form, message=openstack)
else:
return render_template('validation_state.html', state=job.state)
return render_template('validation_state.html', state=job.state)
if __name__ == '__main__':
app.run(host='0.0.0.0', debug=True)