diff --git a/discover_test.py b/discover_test.py index 9bdfa6e..8a72e2b 100644 --- a/discover_test.py +++ b/discover_test.py @@ -6,47 +6,51 @@ from ostack_validator.model import OpenstackComponent from ostack_validator.discovery import OpenstackDiscovery from ostack_validator.inspections import KeystoneAuthtokenSettingsInspection, KeystoneEndpointsInspection -def print_components(openstack): - for host in openstack.hosts: - print('Host %s (id = %s, addresses = %s):' % (host.name, host.id, host.network_addresses)) - for service in host.components: - print('Service %s version %s config %s' % (service.name, service.version, service.config_path)) - service.config +def indent_prefix(indent=0): + s = '' + if indent > 0: + for i in xrange(0, indent): + s += ' ' + return s - # print_service_config(service) +def print_issue(issue, indent=0): + prefix = indent_prefix(indent) -def print_service_config(service): - if isinstance(service, OpenstackComponent): - if service.config: - for section, values in service.config.items(): - print(' [%s]' % section) - for name, value in values.items(): - if value: - print(' %s = %s' % (name, value)) - else: - print('No config file found') + if hasattr(issue, 'mark'): + print('%s[%s] %s (line %d column %d)' % (prefix, issue.type, issue.message, issue.mark.line+1, issue.mark.column+1)) else: - print('Service is not an OpenStack component') - -def print_issues(issues): - # Filer only errors and fatal - issues = [i for i in issues if i.type in [Issue.ERROR, Issue.FATAL]] - - if len(issues) == 0: - print ('No issues found!') - return + print('%s[%s] %s' % (prefix, issue.type, issue.message)) +def print_issues(issues, indent=0): issue_source_f = lambda i: i.mark.source if isinstance(i, MarkedIssue) else None source_groupped_issues = groupby(sorted(issues, key=issue_source_f), key=issue_source_f) - + for source, issues in source_groupped_issues: if source: - print(source) + print('%sFile %s' % (indent_prefix(indent), source)) for issue in sorted(issues, key=lambda i: i.mark.line): - print(' [%s] %s (line %d column %d)' % (issue.type, issue.message, issue.mark.line+1, issue.mark.column+1)) + print_issue(issue, indent+1) else: for issue in issues: - print('[%s] %s' % (issue.type, issue.message)) + print_issue(issue, indent) + +def print_service(service): + print(' ' + str(service)) + print_issues(service.all_issues, indent=2) + +def print_host(host): + print(host) + + print_issues(host.issues, indent=1) + + for service in host.components: + print_service(service) + +def print_openstack(openstack): + print_issues(openstack.issues) + + for host in openstack.hosts: + print_host(host) def main(): logging.basicConfig(level=logging.WARNING) @@ -58,14 +62,12 @@ def main(): openstack = discovery.discover(['172.18.65.179'], 'root', private_key=private_key) - print_components(openstack) - all_inspections = [KeystoneAuthtokenSettingsInspection, KeystoneEndpointsInspection] for inspection in all_inspections: x = inspection() x.inspect(openstack) - print_issues(openstack.all_issues) + print_openstack(openstack) if __name__ == '__main__': main() diff --git a/ostack_validator/discovery.py b/ostack_validator/discovery.py index 1da5b08..8a3f9e0 100644 --- a/ostack_validator/discovery.py +++ b/ostack_validator/discovery.py @@ -7,7 +7,7 @@ import logging import spur from ostack_validator.common import Issue, Mark, MarkedIssue, index, path_relative_to -from ostack_validator.model import Openstack, Host, OpenstackComponent, KeystoneComponent, NovaComputeComponent, GlanceApiComponent +from ostack_validator.model import Openstack, Host, OpenstackComponent, KeystoneComponent, NovaApiComponent, NovaComputeComponent, GlanceApiComponent, GlanceRegistryComponent, MysqlComponent, FileResource @@ -17,7 +17,7 @@ class NodeClient(object): self.shell = spur.SshShell(hostname=node_address, port=ssh_port, username=username, private_key_file=private_key_file, missing_host_key=spur.ssh.MissingHostKey.accept) def run(self, command, *args, **kwargs): - return self.shell.run(command, *args, **kwargs) + return self.shell.run(command, allow_error=True, *args, **kwargs) def open(self, path, mode='r'): return self.shell.open(path, mode) @@ -73,17 +73,24 @@ class OpenstackDiscovery(object): host.id = self._collect_host_id(client) host.network_addresses = self._collect_host_network_addresses(client) - keystone = self._collect_keystone_data(client) - if keystone: - host.add_component(keystone) - - nova_compute = self._collect_nova_data(client) - if nova_compute: - host.add_component(nova_compute) + host.add_component(self._collect_keystone_data(client)) + host.add_component(self._collect_nova_api_data(client)) + host.add_component(self._collect_nova_compute_data(client)) + host.add_component(self._collect_glance_api_data(client)) + host.add_component(self._collect_glance_registry_data(client)) + host.add_component(self._collect_mysql_data(client)) return host + def _find_process(self, client, name): + processes = self._get_processes(client) + for line in processes: + if len(line) > 0 and os.path.basename(line[0]) == name: + return line + + return None + def _find_python_process(self, client, name): processes = self._get_processes(client) for line in processes: @@ -127,6 +134,24 @@ class OpenstackDiscovery(object): addresses.append(match.group(1)) return addresses + def _permissions_string_to_number(self, s): + return 0 + + def _collect_file(self, client, path): + ls = client.run(['ls', '-l', '--time-style=full-iso', path]) + if ls.return_code != 0: + return None + + line = ls.output.split("\n")[0] + perm, links, owner, group, size, date, time, timezone, name = line.split() + permissions = self._permissions_string_to_number(perm) + + with client.open(path) as f: + contents = f.read() + + return FileResource(path, contents, owner, group, permissions) + + def _get_keystone_db_data(self, client, command, env={}): result = client.run(['keystone', command], update_env=env) if result.return_code != 0: @@ -171,10 +196,9 @@ class OpenstackDiscovery(object): else: config_path = '/etc/keystone/keystone.conf' - keystone = KeystoneComponent(config_path) + keystone = KeystoneComponent() keystone.version = self._find_python_package_version(client, 'keystone') - with client.open(config_path) as f: - keystone.config_contents = f.read() + keystone.config_file = self._collect_file(client, config_path) token = keystone.config['DEFAULT']['admin_token'] host = keystone.config['DEFAULT']['bind_host'] @@ -195,25 +219,97 @@ class OpenstackDiscovery(object): return keystone - def _collect_nova_data(self, client): - keystone_process = self._find_python_process(client, 'nova-compute') - if not keystone_process: + def _collect_nova_api_data(self, client): + process = self._find_python_process(client, 'nova-api') + if not process: return None - p = index(keystone_process, lambda s: s == '--config-file') - if p != -1 and p+1 < len(keystone_process): - config_path = keystone_process[p+1] + p = index(process, lambda s: s == '--config-file') + if p != -1 and p+1 < len(process): + config_path = process[p+1] else: config_path = '/etc/nova/nova.conf' - nova_compute = NovaComputeComponent(config_path) - nova_compute.version = self._find_python_package_version(client, 'nova') - with client.open(config_path) as f: - nova_compute.config_contents = f.read() + nova_api = NovaApiComponent() + nova_api.version = self._find_python_package_version(client, 'nova') + nova_api.config_file = self._collect_file(client, config_path) - nova_compute.paste_config_path = path_relative_to(nova_compute.config['DEFAULT']['api_paste_config'], os.path.dirname(config_path)) - with client.open(nova_compute.paste_config_path) as f: - nova_compute.paste_config_contents = f.read() + paste_config_path = path_relative_to(nova_api.config['DEFAULT']['api_paste_config'], os.path.dirname(config_path)) + nova_api.paste_config_file = self._collect_file(client, paste_config_path) + + return nova_api + + def _collect_nova_compute_data(self, client): + process = self._find_python_process(client, 'nova-compute') + if not process: + return None + + p = index(process, lambda s: s == '--config-file') + if p != -1 and p+1 < len(process): + config_path = process[p+1] + else: + config_path = '/etc/nova/nova.conf' + + nova_compute = NovaComputeComponent() + nova_compute.version = self._find_python_package_version(client, 'nova') + nova_compute.config_file = self._collect_file(client, config_path) return nova_compute + def _collect_glance_api_data(self, client): + process = self._find_python_process(client, 'glance-api') + if not process: + return None + + p = index(process, lambda s: s == '--config-file') + if p != -1 and p+1 < len(process): + config_path = process[p+1] + else: + config_path = '/etc/glance/glance-api.conf' + + glance_api = GlanceApiComponent() + glance_api.version = self._find_python_package_version(client, 'glance') + glance_api.config_file = self._collect_file(client, config_path) + + return glance_api + + def _collect_glance_registry_data(self, client): + process = self._find_python_process(client, 'glance-registry') + if not process: + return None + + p = index(process, lambda s: s == '--config-file') + if p != -1 and p+1 < len(process): + config_path = process[p+1] + else: + config_path = '/etc/glance/glance-registry.conf' + + glance_registry = GlanceRegistryComponent() + glance_registry.version = self._find_python_package_version(client, 'glance') + glance_registry.config_file = self._collect_file(client, config_path) + + return glance_registry + + def _collect_mysql_data(self, client): + process = self._find_process(client, 'mysqld') + if not process: + return None + + mysqld_version_re = re.compile('mysqld\s+Ver\s(\S+)\s') + + mysql = MysqlComponent() + + version_result = client.run(['mysqld', '--version']) + m = mysqld_version_re.match(version_result.output) + mysql.version = m.group(1) if m else 'unknown' + + mysql.config_files = [] + config_locations_result = client.run(['bash', '-c', 'mysqld --help --verbose | grep "Default options are read from the following files in the given order" -A 1']) + config_locations = config_locations_result.output.strip().split("\n")[-1].split() + for path in config_locations: + f = self._collect_file(client, path) + if f: + mysql.config_files.append(f) + + return mysql + diff --git a/ostack_validator/inspections/keystone_authtoken.py b/ostack_validator/inspections/keystone_authtoken.py index d4964d3..a44657e 100644 --- a/ostack_validator/inspections/keystone_authtoken.py +++ b/ostack_validator/inspections/keystone_authtoken.py @@ -22,7 +22,7 @@ class KeystoneAuthtokenSettingsInspection(Inspection): if keystone_addresses == ['0.0.0.0']: keystone_addresses = keystone.host.network_addresses - for nova in [c for c in components if c.name == 'nova-compute']: + for nova in [c for c in components if c.name == 'nova-api']: if nova.config['DEFAULT']['auth_strategy'] != 'keystone': continue diff --git a/ostack_validator/model.py b/ostack_validator/model.py index aa87a42..876f259 100644 --- a/ostack_validator/model.py +++ b/ostack_validator/model.py @@ -8,6 +8,7 @@ from ostack_validator.schema import ConfigSchemaRegistry, TypeValidatorRegistry from ostack_validator.config_model import Configuration import ostack_validator.schemas from ostack_validator.config_formats import IniConfigParser +from ostack_validator.utils import memoized class IssueReporter(object): def __init__(self): @@ -15,6 +16,7 @@ class IssueReporter(object): self.issues = [] def report_issue(self, issue): + issue.subject = self self.issues.append(issue) @property @@ -27,6 +29,8 @@ class Openstack(IssueReporter): self.hosts = [] def add_host(self, host): + if not host: return + self.hosts.append(host) host.parent = self @@ -54,7 +58,12 @@ class Host(IssueReporter): self.name = name self.components = [] + def __str__(self): + return 'Host "%s"' % self.name + def add_component(self, component): + if not component: return + self.components.append(component) component.parent = self @@ -88,28 +97,35 @@ class Service(IssueReporter): def openstack(self): return self.host.openstack + @property + def all_issues(self): + result = super(Service, self).all_issues + + if hasattr(self, 'config_file') and self.config_file: + result.extend(self.config_file.all_issues) + + return result + + def __str__(self): + return 'Service "%s"' % self.name + class OpenstackComponent(Service): logger = logging.getLogger('ostack_validator.model.openstack_component') component = None - def __init__(self, config_path): - super(OpenstackComponent, self).__init__() - self.config_path = config_path - self.config_dir = os.path.dirname(config_path) - @property + @memoized def config(self): - if not hasattr(self, '_config'): - schema = ConfigSchemaRegistry.get_schema(self.component, self.version) - if not schema: - self.logger.debug('No schema for component "%s" main config version %s. Skipping it' % (self.component, self.version)) - self._config = None - else: - self._config = self._parse_config_file(Mark(self.config_path), self.config_contents, schema, self) + schema = ConfigSchemaRegistry.get_schema(self.component, self.version) + if not schema: + self.logger.debug('No schema for component "%s" main config version %s. Skipping it' % (self.component, self.version)) + return None - return self._config + return self._parse_config_resource(self.config_file, schema) + def _parse_config_resource(self, resource, schema=None): + return self._parse_config_file(Mark(resource.path), resource.contents, schema, issue_reporter=resource) def _parse_config_file(self, base_mark, config_contents, schema=None, issue_reporter=None): if issue_reporter: @@ -171,6 +187,7 @@ class OpenstackComponent(Service): type_validation_result = type_validator.validate(parameter.value.text) if isinstance(type_validation_result, Issue): type_validation_result.mark = parameter.value.start_mark.merge(type_validation_result.mark) + type_validation_result.message = 'Property "%s" in section "%s": %s' % (parameter.name.text, section_name, type_validation_result.message) report_issue(type_validation_result) else: @@ -194,19 +211,52 @@ class GlanceApiComponent(OpenstackComponent): component = 'glance' name = 'glance-api' +class GlanceRegistryComponent(OpenstackComponent): + component = 'glance' + name = 'glance-registry' + +class NovaApiComponent(OpenstackComponent): + component = 'nova' + name = 'nova-api' + + @property + @memoized + def paste_config(self): + return self._parse_config_resource(self.paste_config_file) + + @property + def all_issues(self): + result = super(NovaApiComponent, self).all_issues + + if hasattr(self, 'paste_config_file') and self.paste_config_file: + result.extend(self.paste_config_file.all_issues) + + return result + + class NovaComputeComponent(OpenstackComponent): component = 'nova' name = 'nova-compute' +class MysqlComponent(Service): + component = 'mysql' + name = 'mysql' + @property - def paste_config(self): - if not hasattr(self, '_paste_config'): - self._paste_config = self._parse_config_file( - Mark(self.paste_config_path), - self.paste_config_contents, - issue_reporter=self - ) + def config_file(self): + if len(self.config_files) == 0: + return None + return self.config_files[0] - return self._paste_config +class FileResource(IssueReporter): + def __init__(self, path, contents, owner, group, permissions): + super(FileResource, self).__init__() + self.path = path + self.contents = contents + self.owner = owner + self.group = group + self.permissions = permissions + def __str__(self): + return 'File "%s"' % self.path diff --git a/ostack_validator/utils.py b/ostack_validator/utils.py new file mode 100644 index 0000000..8e439d5 --- /dev/null +++ b/ostack_validator/utils.py @@ -0,0 +1,29 @@ +import collections +import functools + +class memoized(object): + '''Decorator. Caches a function's return value each time it is called. + If called later with the same arguments, the cached value is returned + (not reevaluated). + ''' + def __init__(self, func): + self.func = func + self.cache = {} + def __call__(self, *args): + if not isinstance(args, collections.Hashable): + # uncacheable. a list, for instance. + # better to not cache than blow up. + return self.func(*args) + if args in self.cache: + return self.cache[args] + else: + value = self.func(*args) + self.cache[args] = value + return value + def __repr__(self): + '''Return the function's docstring.''' + return self.func.__doc__ + def __get__(self, obj, objtype): + '''Support instance methods.''' + return functools.partial(self.__call__, obj) +