Use indexes for lists of modules, companies and users

* Pass record ids from record_filter decorator
* Use record ids to get list of modules, companies and users in
  corresponding drop-downs
* Introduce query filter decorator and use it in drop-down methods

Change-Id: I5fe0e0541162ee57ea423ac5b3aed22c40e2217e

dashboard/decorators.py

@@ -146,6 +146,7 @@ def record_filter(ignore=None, use_default=True):
 
             time_filter = _get_time_filter(kwargs, ignore)
 
+            kwargs['record_ids'] = record_ids
             kwargs['records'] = time_filter(
                 memory_storage_inst.get_records(record_ids))
             return f(*args, **kwargs)
@@ -372,3 +373,21 @@ def jsonify(root='data'):
         return jsonify_decorated_function
 
     return decorator
+
+
+def query_filter(query_param='query'):
+    def decorator(f):
+        @functools.wraps(f)
+        def query_filter_decorated_function(*args, **kwargs):
+
+            query = flask.request.args.get(query_param)
+            if query:
+                kwargs['query_filter'] = lambda x: x.lower().find(query) >= 0
+            else:
+                kwargs['query_filter'] = lambda x: True
+
+            return f(*args, **kwargs)
+
+        return query_filter_decorated_function
+
+    return decorator

dashboard/memory_storage.py

@@ -114,6 +114,11 @@ class CachedMemoryStorage(MemoryStorage):
         return self._get_record_ids_from_index(blueprint_ids,
                                                self.blueprint_id_index)
 
+    def get_index_keys_by_record_ids(self, index_name, record_ids):
+        return set([key
+                    for key, value in six.iteritems(self.indexes[index_name])
+                    if value & record_ids])
+
     def get_record_ids(self):
         return self.records.keys()
 
@@ -133,9 +138,7 @@ class CachedMemoryStorage(MemoryStorage):
 
     def get_original_company_name(self, company_name):
         normalized = company_name.lower()
-        if normalized not in self.company_name_mapping:
+        return self.company_name_mapping.get(normalized, normalized)
-            return normalized
-        return self.company_name_mapping[normalized]
 
     def get_companies(self):
         return self.company_index.keys()

dashboard/reports.py

@@ -226,7 +226,7 @@ def activity():
 @decorators.jsonify('commits')
 @decorators.exception_handler()
 @decorators.record_filter()
-def get_commit_report(records):
+def get_commit_report(records, **kwargs):
     loc_threshold = int(flask.request.args.get('loc_threshold') or 0)
     response = []
     for record in records:
@@ -243,7 +243,7 @@ def get_commit_report(records):
 @decorators.jsonify()
 @decorators.exception_handler()
 @decorators.record_filter(ignore='metric')
-def get_single_plus_two_reviews_report(records):
+def get_single_plus_two_reviews_report(records, **kwargs):
     memory_storage_inst = vault.get_memory_storage()
     plus_twos = collections.defaultdict(list)
     for record in records:

dashboard/web.py

@@ -98,7 +98,7 @@ def _get_aggregated_stats(records, metric_filter, keys, param_id,
 @decorators.jsonify('stats')
 @decorators.exception_handler()
 @decorators.record_filter(ignore='start_date')
-def get_new_companies(records):
+def get_new_companies(records, **kwargs):
 
     days = int(flask.request.args.get('days') or reports.DEFAULT_DAYS_COUNT)
     start_date = int(time.time()) - days * 24 * 60 * 60
@@ -128,7 +128,7 @@ def get_new_companies(records):
 @decorators.exception_handler()
 @decorators.record_filter()
 @decorators.aggregate_filter()
-def get_companies(records, metric_filter, finalize_handler):
+def get_companies(records, metric_filter, finalize_handler, **kwargs):
     return _get_aggregated_stats(records, metric_filter,
                                  vault.get_memory_storage().get_companies(),
                                  'company_name',
@@ -140,7 +140,7 @@ def get_companies(records, metric_filter, finalize_handler):
 @decorators.exception_handler()
 @decorators.record_filter()
 @decorators.aggregate_filter()
-def get_modules(records, metric_filter, finalize_handler):
+def get_modules(records, metric_filter, finalize_handler, **kwargs):
     return _get_aggregated_stats(records, metric_filter,
                                  vault.get_memory_storage().get_modules(),
                                  'module', finalize_handler=finalize_handler)
@@ -161,7 +161,7 @@ def get_core_engineer_branch(user, modules):
 @decorators.exception_handler()
 @decorators.record_filter()
 @decorators.aggregate_filter()
-def get_engineers(records, metric_filter, finalize_handler):
+def get_engineers(records, metric_filter, finalize_handler, **kwargs):
 
     modules_names = parameters.get_parameter({}, 'module', 'modules')
     modules = set([m for m, r in vault.resolve_modules(modules_names, [''])])
@@ -183,7 +183,7 @@ def get_engineers(records, metric_filter, finalize_handler):
 @decorators.jsonify('stats')
 @decorators.exception_handler()
 @decorators.record_filter(ignore='metric')
-def get_engineers_extended(records):
+def get_engineers_extended(records, **kwargs):
     modules_names = parameters.get_parameter({}, 'module', 'modules')
     modules = set([m for m, r in vault.resolve_modules(modules_names, [''])])
 
@@ -228,7 +228,7 @@ def get_engineers_extended(records):
 @decorators.jsonify('stats')
 @decorators.exception_handler()
 @decorators.record_filter()
-def get_distinct_engineers(records):
+def get_distinct_engineers(records, **kwargs):
     result = {}
     for record in records:
         result[record['user_id']] = {
@@ -242,7 +242,7 @@ def get_distinct_engineers(records):
 @decorators.jsonify('activity')
 @decorators.exception_handler()
 @decorators.record_filter()
-def get_activity_json(records):
+def get_activity_json(records, **kwargs):
     start_record = int(flask.request.args.get('start_record') or 0)
     page_size = int(flask.request.args.get('page_size') or
                     parameters.DEFAULT_RECORDS_LIMIT)
@@ -255,7 +255,7 @@ def get_activity_json(records):
 @decorators.jsonify('contribution')
 @decorators.exception_handler()
 @decorators.record_filter(ignore='metric')
-def get_contribution_json(records):
+def get_contribution_json(records, **kwargs):
     return helpers.get_contribution_summary(records)
 
 
@@ -263,32 +263,34 @@ def get_contribution_json(records):
 @decorators.jsonify('companies')
 @decorators.exception_handler()
 @decorators.record_filter(ignore='company')
-def get_companies_json(records):
+@decorators.query_filter(query_param='company_name')
-    query = flask.request.args.get('company_name') or ''
+def get_companies_json(record_ids, query_filter, **kwargs):
-    options = set()
+    memory_storage = vault.get_memory_storage()
-    for record in records:
+    companies = memory_storage.get_index_keys_by_record_ids(
-        name = record['company_name']
+        'company_name', record_ids)
-        if name in options:
+
-            continue
+    result = []
-        if name.lower().find(query.lower()) >= 0:
+    for company in companies:
-            options.add(name)
+        if query_filter(company):
-    result = [{'id': utils.safe_encode(c.lower()), 'text': c}
+            result.append(memory_storage.get_original_company_name(company))
-              for c in sorted(options)]
+
-    return result
+    return [{'id': utils.safe_encode(c.lower()), 'text': c}
+            for c in sorted(result)]
 
 
 @app.route('/api/1.0/modules')
 @decorators.jsonify('modules')
 @decorators.exception_handler()
 @decorators.record_filter(ignore='module')
-def get_modules_json(records):
+@decorators.query_filter(query_param='query')
+def get_modules_json(record_ids, query_filter, **kwargs):
     module_id_index = vault.get_vault()['module_id_index']
 
     tags = parameters.get_parameter({}, 'tag', 'tags')
 
     # all modules mentioned in records
-    module_ids = set(record['module'] for record in records
+    module_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
-                     if record['module'] in module_id_index)
+        'module', record_ids)
 
     add_modules = set([])
     for module in six.itervalues(module_id_index):
@@ -301,19 +303,15 @@ def get_modules_json(records):
         module_ids = set(module_id for module_id in module_ids
                          if module_id_index[module_id].get('tag') in tags)
 
-    query = (flask.request.args.get('query') or '').lower()
+    result = []
-    matched = []
-
     for module_id in module_ids:
-        if module_id.find(query) >= 0:
+        module = module_id_index[module_id]
-            module = dict([(k, v) for k, v in
+        if query_filter(module['module_group_name']):
-                           six.iteritems(module_id_index[module_id])
+            result.append({'id': module['id'],
-                           if k not in ['modules']])
+                           'text': module['module_group_name'],
-            module['text'] = module['module_group_name']
+                           'tag': module['tag']})
-            del module['module_group_name']
-            matched.append(module)
 
-    return sorted(matched, key=operator.itemgetter('text'))
+    return sorted(result, key=operator.itemgetter('text'))
 
 
 @app.route('/api/1.0/companies/<company_name>')
@@ -346,7 +344,7 @@ def get_module(module):
 @decorators.jsonify('members')
 @decorators.exception_handler()
 @decorators.record_filter(ignore=['release', 'project_type', 'module'])
-def get_members(records):
+def get_members(records, **kwargs):
     response = []
     for record in records:
         record = vault.extend_record(record)
@@ -365,7 +363,7 @@ def get_members(records):
 @decorators.jsonify('stats')
 @decorators.exception_handler()
 @decorators.record_filter()
-def get_bpd(records):
+def get_bpd(records, **kwargs):
     result = []
     for record in records:
         if record['record_type'] in ['bpd', 'bpc']:
@@ -395,18 +393,17 @@ def get_bpd(records):
 @decorators.jsonify('users')
 @decorators.exception_handler()
 @decorators.record_filter(ignore='user_id')
-def get_users_json(records):
+@decorators.query_filter(query_param='user_name')
-    user_name_query = flask.request.args.get('user_name') or ''
+def get_users_json(record_ids, query_filter, **kwargs):
-    user_ids = set()
+    user_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
+        'user_id', record_ids)
+
     result = []
-    for record in records:
+    for user_id in user_ids:
-        user_id = record['user_id']
+        user_name = vault.get_user_from_runtime_storage(user_id)['user_name']
-        if user_id in user_ids:
+        if query_filter(user_name):
-            continue
-        user_name = record['author_name']
-        if user_name.lower().find(user_name_query.lower()) >= 0:
-            user_ids.add(user_id)
             result.append({'id': user_id, 'text': user_name})
+
     result.sort(key=lambda x: x['text'])
     return result
 
@@ -424,11 +421,11 @@ def get_user(user_id):
 @app.route('/api/1.0/releases')
 @decorators.jsonify('releases')
 @decorators.exception_handler()
-def get_releases_json():
+@decorators.query_filter(query_param='query')
-    query = (flask.request.args.get('query') or '').lower()
+def get_releases_json(query_filter):
     return [{'id': r['release_name'], 'text': r['release_name'].capitalize()}
             for r in vault.get_release_options()
-            if r['release_name'].find(query) >= 0]
+            if query_filter(r['release_name'])]
 
 
 @app.route('/api/1.0/releases/<release>')
@@ -444,11 +441,11 @@ def get_release_json(release):
 @app.route('/api/1.0/metrics')
 @decorators.jsonify('metrics')
 @decorators.exception_handler()
-def get_metrics_json():
+@decorators.query_filter(query_param='query')
-    query = (flask.request.args.get('query') or '').lower()
+def get_metrics_json(query_filter):
     return sorted([{'id': m, 'text': t}
                    for m, t in six.iteritems(parameters.METRIC_LABELS)
-                   if t.lower().find(query) >= 0],
+                   if query_filter(t)],
                   key=operator.itemgetter('text'))
 
 
@@ -464,9 +461,10 @@ def get_metric_json(metric):
 @app.route('/api/1.0/project_types')
 @decorators.jsonify('project_types')
 @decorators.exception_handler()
-def get_project_types_json():
+@decorators.query_filter(query_param='query')
+def get_project_types_json(query_filter):
     return [{'id': pt['id'], 'text': pt['title'], 'items': pt.get('items', [])}
-            for pt in vault.get_project_types()]
+            for pt in vault.get_project_types() if query_filter(pt['title'])]
 
 
 @app.route('/api/1.0/project_types/<project_type>')

tests/api/test_users.py

@@ -29,7 +29,9 @@ class TestAPIUsers(test_api.TestAPI):
                       'modules': ['nova', 'glance']}],
                  'module_groups': {
                      'nova': test_api.make_module('nova'),
-                     'glance': test_api.make_module('glance')}},
+                     'glance': test_api.make_module('glance')},
+                 'user:john_doe': {'user_name': 'John Doe'},
+                 'user:bill_smith': {'user_name': 'Bill Smith'}},
                 test_api.make_records(record_type=['commit'], module=['nova'],
                                       user_id=['john_doe', 'bill_smith'])):
             response = self.app.get('/api/1.0/users?'
@@ -37,7 +39,7 @@ class TestAPIUsers(test_api.TestAPI):
             users = json.loads(response.data)['users']
             self.assertEqual(2, len(users))
             self.assertIn({'id': 'john_doe', 'text': 'John Doe'}, users)
-            self.assertIn({'id': 'bill_smith', 'text': 'John Doe'}, users)
+            self.assertIn({'id': 'bill_smith', 'text': 'Bill Smith'}, users)
 
     def test_users_search(self):
         with test_api.make_runtime_storage(
@@ -48,7 +50,9 @@ class TestAPIUsers(test_api.TestAPI):
                       'modules': ['nova', 'glance']}],
                  'module_groups': {
                      'nova': test_api.make_module('nova'),
-                     'glance': test_api.make_module('glance')}},
+                     'glance': test_api.make_module('glance')},
+                 'user:john_doe': {'user_name': 'John Doe'},
+                 'user:bill_smith': {'user_name': 'Bill Smith'}},
                 test_api.make_records(record_type=['commit'], module=['nova'],
                                       user_name=['John Doe', 'Bill Smith'])):
             response = self.app.get('/api/1.0/users?'