From 0074cd898fbdc7531eb71d491af0f38409754d95 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Wed, 7 May 2014 18:28:00 +0400 Subject: [PATCH] Use indexes for lists of modules, companies and users * Pass record ids from record_filter decorator * Use record ids to get list of modules, companies and users in corresponding drop-downs * Introduce query filter decorator and use it in drop-down methods Change-Id: I5fe0e0541162ee57ea423ac5b3aed22c40e2217e --- dashboard/decorators.py | 19 +++++++ dashboard/memory_storage.py | 9 ++- dashboard/reports.py | 4 +- dashboard/web.py | 106 ++++++++++++++++++------------------ tests/api/test_users.py | 10 +++- 5 files changed, 86 insertions(+), 62 deletions(-) diff --git a/dashboard/decorators.py b/dashboard/decorators.py index 088e9f641..cf60fc707 100644 --- a/dashboard/decorators.py +++ b/dashboard/decorators.py @@ -146,6 +146,7 @@ def record_filter(ignore=None, use_default=True): time_filter = _get_time_filter(kwargs, ignore) + kwargs['record_ids'] = record_ids kwargs['records'] = time_filter( memory_storage_inst.get_records(record_ids)) return f(*args, **kwargs) @@ -372,3 +373,21 @@ def jsonify(root='data'): return jsonify_decorated_function return decorator + + +def query_filter(query_param='query'): + def decorator(f): + @functools.wraps(f) + def query_filter_decorated_function(*args, **kwargs): + + query = flask.request.args.get(query_param) + if query: + kwargs['query_filter'] = lambda x: x.lower().find(query) >= 0 + else: + kwargs['query_filter'] = lambda x: True + + return f(*args, **kwargs) + + return query_filter_decorated_function + + return decorator diff --git a/dashboard/memory_storage.py b/dashboard/memory_storage.py index 30703169a..b96dc5822 100644 --- a/dashboard/memory_storage.py +++ b/dashboard/memory_storage.py @@ -114,6 +114,11 @@ class CachedMemoryStorage(MemoryStorage): return self._get_record_ids_from_index(blueprint_ids, self.blueprint_id_index) + def get_index_keys_by_record_ids(self, index_name, record_ids): + return set([key + for key, value in six.iteritems(self.indexes[index_name]) + if value & record_ids]) + def get_record_ids(self): return self.records.keys() @@ -133,9 +138,7 @@ class CachedMemoryStorage(MemoryStorage): def get_original_company_name(self, company_name): normalized = company_name.lower() - if normalized not in self.company_name_mapping: - return normalized - return self.company_name_mapping[normalized] + return self.company_name_mapping.get(normalized, normalized) def get_companies(self): return self.company_index.keys() diff --git a/dashboard/reports.py b/dashboard/reports.py index 99890a930..fe312c684 100644 --- a/dashboard/reports.py +++ b/dashboard/reports.py @@ -226,7 +226,7 @@ def activity(): @decorators.jsonify('commits') @decorators.exception_handler() @decorators.record_filter() -def get_commit_report(records): +def get_commit_report(records, **kwargs): loc_threshold = int(flask.request.args.get('loc_threshold') or 0) response = [] for record in records: @@ -243,7 +243,7 @@ def get_commit_report(records): @decorators.jsonify() @decorators.exception_handler() @decorators.record_filter(ignore='metric') -def get_single_plus_two_reviews_report(records): +def get_single_plus_two_reviews_report(records, **kwargs): memory_storage_inst = vault.get_memory_storage() plus_twos = collections.defaultdict(list) for record in records: diff --git a/dashboard/web.py b/dashboard/web.py index 8b13765c7..e81c9c61b 100644 --- a/dashboard/web.py +++ b/dashboard/web.py @@ -98,7 +98,7 @@ def _get_aggregated_stats(records, metric_filter, keys, param_id, @decorators.jsonify('stats') @decorators.exception_handler() @decorators.record_filter(ignore='start_date') -def get_new_companies(records): +def get_new_companies(records, **kwargs): days = int(flask.request.args.get('days') or reports.DEFAULT_DAYS_COUNT) start_date = int(time.time()) - days * 24 * 60 * 60 @@ -128,7 +128,7 @@ def get_new_companies(records): @decorators.exception_handler() @decorators.record_filter() @decorators.aggregate_filter() -def get_companies(records, metric_filter, finalize_handler): +def get_companies(records, metric_filter, finalize_handler, **kwargs): return _get_aggregated_stats(records, metric_filter, vault.get_memory_storage().get_companies(), 'company_name', @@ -140,7 +140,7 @@ def get_companies(records, metric_filter, finalize_handler): @decorators.exception_handler() @decorators.record_filter() @decorators.aggregate_filter() -def get_modules(records, metric_filter, finalize_handler): +def get_modules(records, metric_filter, finalize_handler, **kwargs): return _get_aggregated_stats(records, metric_filter, vault.get_memory_storage().get_modules(), 'module', finalize_handler=finalize_handler) @@ -161,7 +161,7 @@ def get_core_engineer_branch(user, modules): @decorators.exception_handler() @decorators.record_filter() @decorators.aggregate_filter() -def get_engineers(records, metric_filter, finalize_handler): +def get_engineers(records, metric_filter, finalize_handler, **kwargs): modules_names = parameters.get_parameter({}, 'module', 'modules') modules = set([m for m, r in vault.resolve_modules(modules_names, [''])]) @@ -183,7 +183,7 @@ def get_engineers(records, metric_filter, finalize_handler): @decorators.jsonify('stats') @decorators.exception_handler() @decorators.record_filter(ignore='metric') -def get_engineers_extended(records): +def get_engineers_extended(records, **kwargs): modules_names = parameters.get_parameter({}, 'module', 'modules') modules = set([m for m, r in vault.resolve_modules(modules_names, [''])]) @@ -228,7 +228,7 @@ def get_engineers_extended(records): @decorators.jsonify('stats') @decorators.exception_handler() @decorators.record_filter() -def get_distinct_engineers(records): +def get_distinct_engineers(records, **kwargs): result = {} for record in records: result[record['user_id']] = { @@ -242,7 +242,7 @@ def get_distinct_engineers(records): @decorators.jsonify('activity') @decorators.exception_handler() @decorators.record_filter() -def get_activity_json(records): +def get_activity_json(records, **kwargs): start_record = int(flask.request.args.get('start_record') or 0) page_size = int(flask.request.args.get('page_size') or parameters.DEFAULT_RECORDS_LIMIT) @@ -255,7 +255,7 @@ def get_activity_json(records): @decorators.jsonify('contribution') @decorators.exception_handler() @decorators.record_filter(ignore='metric') -def get_contribution_json(records): +def get_contribution_json(records, **kwargs): return helpers.get_contribution_summary(records) @@ -263,32 +263,34 @@ def get_contribution_json(records): @decorators.jsonify('companies') @decorators.exception_handler() @decorators.record_filter(ignore='company') -def get_companies_json(records): - query = flask.request.args.get('company_name') or '' - options = set() - for record in records: - name = record['company_name'] - if name in options: - continue - if name.lower().find(query.lower()) >= 0: - options.add(name) - result = [{'id': utils.safe_encode(c.lower()), 'text': c} - for c in sorted(options)] - return result +@decorators.query_filter(query_param='company_name') +def get_companies_json(record_ids, query_filter, **kwargs): + memory_storage = vault.get_memory_storage() + companies = memory_storage.get_index_keys_by_record_ids( + 'company_name', record_ids) + + result = [] + for company in companies: + if query_filter(company): + result.append(memory_storage.get_original_company_name(company)) + + return [{'id': utils.safe_encode(c.lower()), 'text': c} + for c in sorted(result)] @app.route('/api/1.0/modules') @decorators.jsonify('modules') @decorators.exception_handler() @decorators.record_filter(ignore='module') -def get_modules_json(records): +@decorators.query_filter(query_param='query') +def get_modules_json(record_ids, query_filter, **kwargs): module_id_index = vault.get_vault()['module_id_index'] tags = parameters.get_parameter({}, 'tag', 'tags') # all modules mentioned in records - module_ids = set(record['module'] for record in records - if record['module'] in module_id_index) + module_ids = vault.get_memory_storage().get_index_keys_by_record_ids( + 'module', record_ids) add_modules = set([]) for module in six.itervalues(module_id_index): @@ -301,19 +303,15 @@ def get_modules_json(records): module_ids = set(module_id for module_id in module_ids if module_id_index[module_id].get('tag') in tags) - query = (flask.request.args.get('query') or '').lower() - matched = [] - + result = [] for module_id in module_ids: - if module_id.find(query) >= 0: - module = dict([(k, v) for k, v in - six.iteritems(module_id_index[module_id]) - if k not in ['modules']]) - module['text'] = module['module_group_name'] - del module['module_group_name'] - matched.append(module) + module = module_id_index[module_id] + if query_filter(module['module_group_name']): + result.append({'id': module['id'], + 'text': module['module_group_name'], + 'tag': module['tag']}) - return sorted(matched, key=operator.itemgetter('text')) + return sorted(result, key=operator.itemgetter('text')) @app.route('/api/1.0/companies/') @@ -346,7 +344,7 @@ def get_module(module): @decorators.jsonify('members') @decorators.exception_handler() @decorators.record_filter(ignore=['release', 'project_type', 'module']) -def get_members(records): +def get_members(records, **kwargs): response = [] for record in records: record = vault.extend_record(record) @@ -365,7 +363,7 @@ def get_members(records): @decorators.jsonify('stats') @decorators.exception_handler() @decorators.record_filter() -def get_bpd(records): +def get_bpd(records, **kwargs): result = [] for record in records: if record['record_type'] in ['bpd', 'bpc']: @@ -395,18 +393,17 @@ def get_bpd(records): @decorators.jsonify('users') @decorators.exception_handler() @decorators.record_filter(ignore='user_id') -def get_users_json(records): - user_name_query = flask.request.args.get('user_name') or '' - user_ids = set() +@decorators.query_filter(query_param='user_name') +def get_users_json(record_ids, query_filter, **kwargs): + user_ids = vault.get_memory_storage().get_index_keys_by_record_ids( + 'user_id', record_ids) + result = [] - for record in records: - user_id = record['user_id'] - if user_id in user_ids: - continue - user_name = record['author_name'] - if user_name.lower().find(user_name_query.lower()) >= 0: - user_ids.add(user_id) + for user_id in user_ids: + user_name = vault.get_user_from_runtime_storage(user_id)['user_name'] + if query_filter(user_name): result.append({'id': user_id, 'text': user_name}) + result.sort(key=lambda x: x['text']) return result @@ -424,11 +421,11 @@ def get_user(user_id): @app.route('/api/1.0/releases') @decorators.jsonify('releases') @decorators.exception_handler() -def get_releases_json(): - query = (flask.request.args.get('query') or '').lower() +@decorators.query_filter(query_param='query') +def get_releases_json(query_filter): return [{'id': r['release_name'], 'text': r['release_name'].capitalize()} for r in vault.get_release_options() - if r['release_name'].find(query) >= 0] + if query_filter(r['release_name'])] @app.route('/api/1.0/releases/') @@ -444,11 +441,11 @@ def get_release_json(release): @app.route('/api/1.0/metrics') @decorators.jsonify('metrics') @decorators.exception_handler() -def get_metrics_json(): - query = (flask.request.args.get('query') or '').lower() +@decorators.query_filter(query_param='query') +def get_metrics_json(query_filter): return sorted([{'id': m, 'text': t} for m, t in six.iteritems(parameters.METRIC_LABELS) - if t.lower().find(query) >= 0], + if query_filter(t)], key=operator.itemgetter('text')) @@ -464,9 +461,10 @@ def get_metric_json(metric): @app.route('/api/1.0/project_types') @decorators.jsonify('project_types') @decorators.exception_handler() -def get_project_types_json(): +@decorators.query_filter(query_param='query') +def get_project_types_json(query_filter): return [{'id': pt['id'], 'text': pt['title'], 'items': pt.get('items', [])} - for pt in vault.get_project_types()] + for pt in vault.get_project_types() if query_filter(pt['title'])] @app.route('/api/1.0/project_types/') diff --git a/tests/api/test_users.py b/tests/api/test_users.py index 89d035c5c..9d580abd7 100644 --- a/tests/api/test_users.py +++ b/tests/api/test_users.py @@ -29,7 +29,9 @@ class TestAPIUsers(test_api.TestAPI): 'modules': ['nova', 'glance']}], 'module_groups': { 'nova': test_api.make_module('nova'), - 'glance': test_api.make_module('glance')}}, + 'glance': test_api.make_module('glance')}, + 'user:john_doe': {'user_name': 'John Doe'}, + 'user:bill_smith': {'user_name': 'Bill Smith'}}, test_api.make_records(record_type=['commit'], module=['nova'], user_id=['john_doe', 'bill_smith'])): response = self.app.get('/api/1.0/users?' @@ -37,7 +39,7 @@ class TestAPIUsers(test_api.TestAPI): users = json.loads(response.data)['users'] self.assertEqual(2, len(users)) self.assertIn({'id': 'john_doe', 'text': 'John Doe'}, users) - self.assertIn({'id': 'bill_smith', 'text': 'John Doe'}, users) + self.assertIn({'id': 'bill_smith', 'text': 'Bill Smith'}, users) def test_users_search(self): with test_api.make_runtime_storage( @@ -48,7 +50,9 @@ class TestAPIUsers(test_api.TestAPI): 'modules': ['nova', 'glance']}], 'module_groups': { 'nova': test_api.make_module('nova'), - 'glance': test_api.make_module('glance')}}, + 'glance': test_api.make_module('glance')}, + 'user:john_doe': {'user_name': 'John Doe'}, + 'user:bill_smith': {'user_name': 'Bill Smith'}}, test_api.make_records(record_type=['commit'], module=['nova'], user_name=['John Doe', 'Bill Smith'])): response = self.app.get('/api/1.0/users?'