Discovery API improvements, PEP8 issues clean up

Previous discovery API didn't allowed agents to get dependent
resources synchronously, e.g. when discovering some service,
get service's configuration file via FileDiscovery agent and
have it cached so that subsequent discovery of that file would
not produce a new resource.
New API changes how discovery is done: agents now represent a
caching resource factories, so when you ask for particular
resource for the second time, it will return a cached instance.
Resources no more accumulated in the main loop but instead collected
from agents afterwards. This overcomes the fact that some agent
could produce multiple resources while current API allows only one
resource to be returned from agent's discover() method.

Extended file discovery methods: now you can use wildcards to collect
multiple files at once and use search paths (e.g. when you want to
search for config file in several directories and collect whichever
found first).

Cleaned up PEP8 issues regarding unused imports and 'import *'.

Added CLI options for discovery_test to output JSON and choose log
level.

IssueReporter subclasses now won't contain duplicate entries.

Change-Id: I7371ccce1e2f3c0a649fe9d6a41b146f04c0f4c1

discover_test.py

@@ -1,3 +1,4 @@
+import argparse
 from flask import json
 from itertools import groupby
 import logging
@@ -5,13 +6,9 @@ import sys
 
 from rubick.common import MarkedIssue, Inspection
 from rubick.discovery import OpenstackDiscovery
-import rubick.inspections
-# Silence PEP8 "unused import"
-assert rubick.inspections
-import rubick.schemas
-assert rubick.schemas
+import rubick.inspections     # noqa
+import rubick.schemas         # noqa
 from rubick.json import openstack_for_json
-from rubick.model import DirectoryResource
 
 
 def indent_prefix(indent=0):
@@ -52,14 +49,11 @@ def print_issues(issues, indent=0):
 
 def print_service(service):
     print('    ' + service.name)
-    print_issues(service.all_issues, indent=3)
+    print_issues(service.issues, indent=3)
 
 
 def print_path(path):
-    if isinstance(path, DirectoryResource):
-        print('    ' + path.path + '/')
-    else:
-        print('    ' + path.path)
+    print('    ' + path.path)
     print_issues(path.all_issues, indent=3)
 
 
@@ -70,7 +64,7 @@ def print_host(host):
 
     print('  Services:')
 
-    for service in host.components:
+    for service in sorted(host.components, key=lambda c: c.name):
         print_service(service)
 
     print('  Filesystem:')
@@ -86,9 +80,23 @@ def print_openstack(openstack):
         print_host(host)
 
 
-def main():
+def parse_args(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-l', '--loglevel', default='INFO',
+                        help='Loglevel to use')
+    parser.add_argument('-j', '--json', dest='json', default=False,
+                        action='store_true',
+                        help='Output result in JSON format')
+    args = parser.parse_args(argv[1:])
+    return args
+
+
+def main(argv):
+    args = parse_args(argv)
+    params = vars(args)
+
     logging.basicConfig(level=logging.WARNING)
-    logging.getLogger('rubick').setLevel(logging.DEBUG)
+    logging.getLogger('rubick').setLevel(params['loglevel'])
 
     discovery = OpenstackDiscovery()
     try:
@@ -106,8 +114,10 @@ def main():
         x = inspection()
         x.inspect(openstack)
 
-    print_openstack(openstack)
-    # print(json.dumps(openstack_for_json(openstack)))
+    if params['json']:
+        print(json.dumps(openstack_for_json(openstack)))
+    else:
+        print_openstack(openstack)
 
 if __name__ == '__main__':
-    main()
+    main(sys.argv)

joker/nodes.py

@@ -5,7 +5,6 @@ from paramiko.dsskey import DSSKey
 from paramiko.rsakey import RSAKey
 from six import StringIO
 import stat
-import sys
 
 TMP_KEY_PATH = "/tmp/joker_%s_%d"
 

joker/ssh.py

@@ -1,6 +1,6 @@
 from os import environ
-import shlex
-import subprocess
+#import shlex
+#import subprocess
 
 
 class JokerSecureShell():

requirements.txt

@@ -16,4 +16,4 @@ paramiko==1.11.0
 oslo.config==1.2.1
 requests==1.2.0
 PyYAML==3.10
-six==1.4.1
+six>=1.4.1

rubick/common.py

@@ -170,6 +170,15 @@ class Issue(object):
         self.type = type
         self.message = message
 
+    def __eq__(self, other):
+        if not isinstance(other, Issue):
+            return False
+
+        return self.type == other.type and self.message == other.message
+
+    def __ne__(self, other):
+        return not self == other
+
     def __repr__(self):
         return (
             '<%s type=%s message=%s>' % (
@@ -193,6 +202,15 @@ class MarkedIssue(Issue):
         other.mark = base_mark.merge(self.mark)
         return other
 
+    def __eq__(self, other):
+        if not isinstance(other, MarkedIssue):
+            return False
+
+        return super(MarkedIssue, self).__eq__(other) and self.mark == other.mark
+
+    def __ne__(self, other):
+        return not self == other
+
     def __repr__(self):
         return (
             '<%s type=%s message=%s mark=%s>' % (

rubick/config_formats/__init__.py

@@ -1,2 +1 @@
-from rubick.common import *
-from rubick.config_formats.ini import IniConfigParser
+from rubick.config_formats.ini import IniConfigParser  # noqa

rubick/config_formats/common.py

@@ -1,4 +1,4 @@
-from rubick.common import Mark, Issue, MarkedIssue
+from rubick.common import Issue, MarkedIssue
 
 
 class ParseError(MarkedIssue):

rubick/config_formats/ini.py

@@ -1,8 +1,12 @@
 import re
+
 from six import StringIO
 
-from rubick.config_model import *
-from rubick.config_formats.common import *
+from rubick.common import Mark
+from rubick.config_model import ComponentConfig, ConfigSection, \
+    ConfigSectionName, ConfigParameter, ConfigParameterName, \
+    ConfigParameterValue, TextElement
+from rubick.config_formats.common import ParseError
 
 
 class IniConfigParser:

rubick/config_formats/test_ini.py

@@ -1,6 +1,9 @@
 import unittest
 
-from rubick.config_formats.ini import *
+from six import StringIO
+
+from rubick.common import Mark
+from rubick.config_formats.ini import IniConfigParser
 
 
 class IniConfigParserTests(unittest.TestCase):

rubick/inspections/__init__.py

@@ -1,4 +1,3 @@
-from rubick.inspections.keystone_authtoken \
-    import KeystoneAuthtokenSettingsInspection
-from rubick.inspections.keystone_endpoints import KeystoneEndpointsInspection
-from rubick.inspections.lettuce_runner import LettuceRunnerInspection
+from rubick.inspections.keystone_authtoken import KeystoneAuthtokenSettingsInspection  # noqa
+from rubick.inspections.keystone_endpoints import KeystoneEndpointsInspection          # noqa
+from rubick.inspections.lettuce_runner import LettuceRunnerInspection                  # noqa

rubick/inspections/lettuce/steps.py

@@ -2,7 +2,11 @@ import string
 from lettuce import step, world
 
 from rubick.common import Issue, Version, find
-from rubick.model import *
+import rubick.model as model
+
+
+AUTHTOKEN_FILTER_FACTORY = ('keystoneclient.middleware.auth_token:'
+                            'filter_factory')
 
 
 def get_variable(name):
@@ -36,7 +40,7 @@ def stop():
 def use_openstack_version(step, version):
     version = Version(version)
     for component in [c for c in world.openstack.components
-                      if isinstance(c, OpenstackComponent)]:
+                      if isinstance(c, model.OpenstackComponent)]:
         if not Version(component.version) >= version:
             stop()
 

rubick/model.py

@@ -15,8 +15,9 @@ class IssueReporter(object):
         self.issues = []
 
     def report_issue(self, issue):
-        issue.subject = self
-        self.issues.append(issue)
+        if issue not in self.issues:
+            issue.subject = self
+            self.issues.append(issue)
 
     @property
     def all_issues(self):

tox.ini

@@ -32,7 +32,9 @@ commands =
 commands = {posargs}
 
 [flake8]
-# TODO(ogelbukh): enforce F401, F403 and H306
+# TODO(ogelbukh): enforce H306
 # We will be ignoring line length limit until we apply for incubation
-ignore = F401,F403,E501,H306
+# E501: line too long
+# H306: imports not in alphabetical order
+ignore = E501,H306
 exclude =  .venv,.git,.tox,dist,lib/python*,*egg,build