diff --git a/dashboard/decorators.py b/dashboard/decorators.py index f301ef5d9..c012a21e9 100644 --- a/dashboard/decorators.py +++ b/dashboard/decorators.py @@ -58,7 +58,6 @@ def record_filter(ignore=None, use_default=True): @functools.wraps(f) def record_filter_decorated_function(*args, **kwargs): - vault_inst = vault.get_vault() memory_storage_inst = vault.get_memory_storage() record_ids = set(memory_storage_inst.get_record_ids()) # a copy @@ -70,19 +69,6 @@ def record_filter(ignore=None, use_default=True): memory_storage_inst.get_record_ids_by_modules( vault.resolve_modules(param))) - if 'project_type' not in ignore: - param = parameters.get_parameter(kwargs, 'project_type', - 'project_types', use_default) - if param: - ptgi = vault_inst['project_type_group_index'] - modules = set() - for project_type in param: - project_type = project_type.lower() - if project_type in ptgi: - modules |= ptgi[project_type] - record_ids &= ( - memory_storage_inst.get_record_ids_by_modules(modules)) - if 'user_id' not in ignore: param = parameters.get_parameter(kwargs, 'user_id', 'user_ids') param = [u for u in param @@ -265,11 +251,6 @@ def templated(template=None, return_code=200): ctx['metric'] = metric or parameters.get_default('metric') ctx['metric_label'] = parameters.METRIC_LABELS[ctx['metric']] - project_type = flask.request.args.get('project_type') - if not vault.is_project_type_valid(project_type): - project_type = parameters.get_default('project_type') - ctx['project_type'] = project_type - release = flask.request.args.get('release') releases = vault_inst['releases'] if release: @@ -284,7 +265,6 @@ def templated(template=None, return_code=200): ctx['review_nth'] = (flask.request.args.get('review_nth') or parameters.get_default('review_nth')) - ctx['project_type_options'] = vault.get_project_type_options() ctx['release_options'] = vault.get_release_options() ctx['metric_options'] = sorted(parameters.METRIC_LABELS.items(), key=lambda x: x[0]) @@ -293,7 +273,11 @@ def templated(template=None, return_code=200): ctx['company_original'] = ( vault.get_memory_storage().get_original_company_name( ctx['company'])) - ctx['module'] = parameters.get_single_parameter(kwargs, 'module') + + module = parameters.get_single_parameter(kwargs, 'module') + ctx['module'] = module + ctx['module_inst'] = vault_inst['module_id_index'][module] + ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id') ctx['page_title'] = helpers.make_page_title( ctx['company'], ctx['user_id'], ctx['module'], ctx['release']) diff --git a/dashboard/helpers.py b/dashboard/helpers.py index d8fc59c88..462f602b7 100644 --- a/dashboard/helpers.py +++ b/dashboard/helpers.py @@ -21,7 +21,6 @@ import six from dashboard import parameters from dashboard import vault -from stackalytics.openstack.common.py3kcompat import urlutils from stackalytics.processor import utils @@ -178,13 +177,8 @@ def format_launchpad_module_link(module): return '%s' % (module, module) -def safe_encode(s): - return urlutils.quote_plus(s.encode('utf-8')) - - def make_link(title, uri=None, options=None): - param_names = ('release', 'project_type', 'module', 'company', 'user_id', - 'metric') + param_names = ('release', 'module', 'company', 'user_id', 'metric') param_values = {} for param_name in param_names: v = parameters.get_parameter({}, param_name, param_name) @@ -193,7 +187,7 @@ def make_link(title, uri=None, options=None): if options: param_values.update(options) if param_values: - uri += '?' + '&'.join(['%s=%s' % (n, safe_encode(v)) + uri += '?' + '&'.join(['%s=%s' % (n, utils.safe_encode(v)) for n, v in six.iteritems(param_values)]) return '%(title)s' % {'uri': uri, 'title': title} diff --git a/dashboard/parameters.py b/dashboard/parameters.py index 85fafce35..8a6091e57 100644 --- a/dashboard/parameters.py +++ b/dashboard/parameters.py @@ -25,7 +25,7 @@ LOG = logging.getLogger(__name__) DEFAULTS = { 'metric': 'commits', 'release': 'icehouse', - 'project_type': 'openstack', + 'module': 'openstack', 'review_nth': 5, } diff --git a/dashboard/static/js/stackalytics-ui.js b/dashboard/static/js/stackalytics-ui.js index f4ec09975..3d56b1951 100644 --- a/dashboard/static/js/stackalytics-ui.js +++ b/dashboard/static/js/stackalytics-ui.js @@ -276,7 +276,6 @@ function make_std_options() { var options = {}; options['release'] = $('#release').val(); options['metric'] = $('#metric').val(); - options['project_type'] = $('#project_type').val(); options['module'] = $('#module').val() || ''; options['company'] = $('#company').val() || ''; options['user_id'] = $('#user').val() || ''; @@ -436,7 +435,7 @@ function init_selectors(base_url) { }); $("#module").select2({ - allowClear: true, + allowClear: false, ajax: { url: make_uri(base_url + "/api/1.0/modules"), dataType: 'jsonp', diff --git a/dashboard/templates/kpi/example.html b/dashboard/templates/kpi/example.html index 41160004f..6166e90db 100644 --- a/dashboard/templates/kpi/example.html +++ b/dashboard/templates/kpi/example.html @@ -8,18 +8,18 @@ diff --git a/dashboard/templates/layout.html b/dashboard/templates/layout.html index c4e45850b..48f252b56 100644 --- a/dashboard/templates/layout.html +++ b/dashboard/templates/layout.html @@ -59,7 +59,7 @@ About ↗
- + | community heartbeat
@@ -67,32 +67,27 @@
- -
- -
- - +
- +
- +
- +
- +
diff --git a/dashboard/templates/overview.html b/dashboard/templates/overview.html index e535c08a8..c7c9e6187 100644 --- a/dashboard/templates/overview.html +++ b/dashboard/templates/overview.html @@ -4,9 +4,9 @@ {% set show_company_breakdown = (not company) and (not user_id) %} {% set show_engineer_breakdown = (not user_id) %} {% set show_bp_breakdown = (metric in ['bpd', 'bpc']) %} -{% set show_module_breakdown = (not module) %} +{% set show_module_breakdown = (not module) or (module_inst['group']) %} {% set show_user_activity = (user_id) %} -{% set show_module_activity = (module) and (not user_id) %} +{% set show_module_activity = (module) and (not user_id) and (not module_inst['group']) %} {% set show_activity = (show_user_activity) or (show_module_activity) %} {% set show_user_contribution = (user_id) or (company) %} {% set show_module_contribution = (module) and (not user_id) %} diff --git a/dashboard/templates/reports/contribution.html b/dashboard/templates/reports/contribution.html index 008e10d62..dd7e6cefa 100644 --- a/dashboard/templates/reports/contribution.html +++ b/dashboard/templates/reports/contribution.html @@ -12,7 +12,7 @@ Contribution into {{ module }} for the last {{ days }} days var table_id = "review_stats_table"; $.ajax({ - url: make_uri("/api/1.0/stats/engineers_extended?project_type=all&module={{ module }}&release=all&start_date={{ start_date }}"), + url: make_uri("/api/1.0/stats/engineers_extended?metric=marks&module={{ module }}&release=all&start_date={{ start_date }}"), dataType: "json", success: function (data) { var tableData = data["stats"]; diff --git a/dashboard/templates/widget.html b/dashboard/templates/widget.html index e9a178999..c45688ddd 100644 --- a/dashboard/templates/widget.html +++ b/dashboard/templates/widget.html @@ -121,8 +121,8 @@
- - + +
diff --git a/dashboard/vault.py b/dashboard/vault.py index 7edd076c0..0858609a2 100644 --- a/dashboard/vault.py +++ b/dashboard/vault.py @@ -17,7 +17,6 @@ import os import flask from oslo.config import cfg -import six from dashboard import memory_storage from stackalytics.openstack.common import log as logging @@ -39,7 +38,6 @@ def get_vault(): vault['memory_storage'] = memory_storage.get_memory_storage( memory_storage.MEMORY_STORAGE_CACHED) - init_project_types(vault) init_releases(vault) flask.current_app.stackalytics_vault = vault @@ -55,7 +53,6 @@ def get_vault(): vault['runtime_storage'].get_update(os.getpid())) if have_updates: - init_project_types(vault) init_releases(vault) init_module_groups(vault) @@ -84,53 +81,20 @@ def init_releases(vault): vault['releases'] = releases_map -def init_project_types(vault): - runtime_storage_inst = vault['runtime_storage'] - project_type_options = {} - project_type_group_index = {'all': set(['unknown'])} - - for repo in utils.load_repos(runtime_storage_inst): - project_type = repo['project_type'].lower() - project_group = None - if ('project_group' in repo) and (repo['project_group']): - project_group = repo['project_group'].lower() - - if project_type in project_type_options: - if project_group: - project_type_options[project_type].add(project_group) - else: - if project_group: - project_type_options[project_type] = set([project_group]) - else: - project_type_options[project_type] = set() - - module = repo['module'] - if project_type in project_type_group_index: - project_type_group_index[project_type].add(module) - else: - project_type_group_index[project_type] = set([module]) - - if project_group: - if project_group in project_type_group_index: - project_type_group_index[project_group].add(module) - else: - project_type_group_index[project_group] = set([module]) - - project_type_group_index['all'].add(module) - - vault['project_type_options'] = project_type_options - vault['project_type_group_index'] = project_type_group_index - - def init_module_groups(vault): runtime_storage_inst = vault['runtime_storage'] + memory_storage_inst = vault['memory_storage'] + module_index = {} module_id_index = {} module_groups = runtime_storage_inst.get_by_key('module_groups') or [] + module_groups.append({'module_group_name': 'All', + 'modules': set(memory_storage_inst.get_modules())}) + for module_group in module_groups: module_group_name = module_group['module_group_name'] - module_group_id = module_group_name.lower() + module_group_id = utils.safe_encode(module_group_name.lower()) module_id_index[module_group_id] = { 'group': True, @@ -146,10 +110,9 @@ def init_module_groups(vault): else: module_index[module] = set([module_group_id]) - memory_storage_inst = vault['memory_storage'] for module in memory_storage_inst.get_modules(): module_id_index[module] = { - 'id': module.lower(), + 'id': utils.safe_encode(module.lower()), 'text': module, 'modules': [module.lower()], } @@ -159,10 +122,6 @@ def init_module_groups(vault): vault['module_groups'] = module_groups -def get_project_type_options(): - return get_vault()['project_type_options'] - - def get_release_options(): runtime_storage_inst = get_vault()['runtime_storage'] releases = (runtime_storage_inst.get_by_key('releases') or [None])[1:] @@ -171,28 +130,13 @@ def get_release_options(): return releases -def is_project_type_valid(project_type): - if not project_type: - return False - project_type = project_type.lower() - if project_type == 'all': - return True - project_types = get_project_type_options() - if project_type in project_types: - return True - for p, g in six.iteritems(project_types): - if project_type in g: - return True - return False - - def get_user_from_runtime_storage(user_id): runtime_storage_inst = get_vault()['runtime_storage'] return utils.load_user(runtime_storage_inst, user_id) def resolve_modules(module_ids): - module_id_index = get_vault()['module_id_index'] + module_id_index = get_vault().get('module_id_index') or {} modules = set() for module_id in module_ids: if module_id in module_id_index: diff --git a/dashboard/web.py b/dashboard/web.py index 2da2d8771..3aaf4e336 100644 --- a/dashboard/web.py +++ b/dashboard/web.py @@ -242,7 +242,7 @@ def get_companies_json(records): continue if name.lower().find(query.lower()) >= 0: options.add(name) - result = [{'id': helpers.safe_encode(c.lower()), 'text': c} + result = [{'id': utils.safe_encode(c.lower()), 'text': c} for c in sorted(options)] return result @@ -406,7 +406,7 @@ def get_metric_json(metric): @decorators.exception_handler() def get_project_types_json(): return [{'id': m, 'text': m, 'items': list(t)} - for m, t in six.iteritems(vault.get_project_type_options())] + for m, t in vault.get_project_type_options().iteritems()] @app.route('/api/1.0/project_types/') @@ -414,7 +414,7 @@ def get_project_types_json(): @decorators.exception_handler() def get_project_type_json(project_type): if project_type != 'all': - for pt, groups in six.iteritems(vault.get_project_type_options()): + for pt, groups in vault.get_project_type_options().iteritems(): if (project_type == pt) or (project_type in groups): break else: diff --git a/etc/default_data.json b/etc/default_data.json index dd2c173f1..ddff3b448 100644 --- a/etc/default_data.json +++ b/etc/default_data.json @@ -323,7 +323,7 @@ } ], "user_name": "armando-migliaccio", - "emails": ["amigliaccio@nicira.com", "Armando.Migliaccio@eu.citrix.com", "amigliaccio@internap.com", "armamig@gmail.com", "armando.migliaccio@citrix.com", "armando.migliaccio@eu.citrix.com"] + "emails": ["amigliaccio@nicira.com", "armando.migliaccio@eu.citrix.com", "amigliaccio@internap.com", "armamig@gmail.com", "armando.migliaccio@citrix.com"] }, { "launchpad_id": "arnaudleg", @@ -2303,7 +2303,7 @@ } ], "user_name": "Juan Perez", - "emails": ["Juan_Perez@cable.comcast.com"] + "emails": ["juan_perez@cable.comcast.com"] }, { "launchpad_id": "jun-park-earth", @@ -3104,7 +3104,7 @@ } ], "user_name": "Zhenguo Niu", - "emails": ["zhenguo@unitedstack.com", "Niu.ZGlinux@gmail.com"] + "emails": ["zhenguo@unitedstack.com", "niu.zglinux@gmail.com"] }, { "launchpad_id": "nobodycam", @@ -3692,7 +3692,7 @@ } ], "user_name": "Sean Gossard", - "emails": ["sgossard@parelastic.com", "sgossard@tesora.com", "seanGossard@usa.com"] + "emails": ["sgossard@parelastic.com", "sgossard@tesora.com", "seangossard@usa.com"] }, { "launchpad_id": "sean-mccully", @@ -4292,7 +4292,7 @@ } ], "user_name": "Trevor R Jr", - "emails": ["vmtrooper@gmail.com", "Trevor.RobertsJr@vce.com"] + "emails": ["vmtrooper@gmail.com", "trevor.robertsjr@vce.com"] }, { "launchpad_id": "vsharshov", @@ -5295,7 +5295,6 @@ ], "repos": [ { - "project_group": "core", "releases": [ { "tag_to": "2012.1", @@ -5328,11 +5327,9 @@ ], "uri": "git://github.com/openstack/nova.git", "module": "nova", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "2012.1", @@ -5365,11 +5362,9 @@ ], "uri": "git://github.com/openstack/keystone.git", "module": "keystone", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "branch": "stable/folsom", @@ -5397,11 +5392,9 @@ ], "uri": "git://github.com/openstack/cinder.git", "module": "cinder", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "2012.1", @@ -5434,11 +5427,9 @@ ], "uri": "git://github.com/openstack/glance.git", "module": "glance", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "2012.1", @@ -5471,11 +5462,9 @@ ], "uri": "git://github.com/openstack/neutron.git", "module": "neutron", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "2012.1", @@ -5508,11 +5497,9 @@ ], "uri": "git://github.com/openstack/horizon.git", "module": "horizon", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "1.4.8", @@ -5545,11 +5532,9 @@ ], "uri": "git://github.com/openstack/swift.git", "module": "swift", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "0.2.0", @@ -5574,11 +5559,9 @@ ], "uri": "git://github.com/openstack/python-keystoneclient.git", "module": "python-keystoneclient", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "2.9.0", @@ -5603,11 +5586,9 @@ ], "uri": "git://github.com/openstack/python-novaclient.git", "module": "python-novaclient", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "1.0.3", @@ -5627,11 +5608,9 @@ ], "uri": "git://github.com/openstack/python-cinderclient.git", "module": "python-cinderclient", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "0.9.0", @@ -5651,11 +5630,9 @@ ], "uri": "git://github.com/openstack/python-glanceclient.git", "module": "python-glanceclient", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "2.1", @@ -5680,11 +5657,9 @@ ], "uri": "git://github.com/openstack/python-neutronclient.git", "module": "python-neutronclient", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "1.3.0", @@ -5704,11 +5679,9 @@ ], "uri": "git://github.com/openstack/python-swiftclient.git", "module": "python-swiftclient", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "branch": "stable/grizzly", @@ -5730,11 +5703,9 @@ ], "uri": "git://github.com/openstack/heat.git", "module": "heat", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "0.2.2", @@ -5754,11 +5725,9 @@ ], "uri": "git://github.com/openstack/python-heatclient.git", "module": "python-heatclient", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "branch": "stable/grizzly", @@ -5780,11 +5749,9 @@ ], "uri": "git://github.com/openstack/ceilometer.git", "module": "ceilometer", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "tag_to": "1.0.6", @@ -5799,11 +5766,9 @@ ], "uri": "git://github.com/openstack/python-ceilometerclient.git", "module": "python-ceilometerclient", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "branch": "stable/havana", @@ -5819,11 +5784,9 @@ ], "uri": "git://github.com/openstack/oslo-incubator.git", "module": "oslo-incubator", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "core", "releases": [ { "branch": "stable/havana", @@ -5839,11 +5802,9 @@ ], "uri": "git://github.com/openstack/oslo.config.git", "module": "oslo.config", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "documentation", "releases": [ { "tag_to": "HEAD", @@ -5858,11 +5819,9 @@ ], "uri": "git://github.com/openstack/compute-api.git", "module": "compute-api", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "documentation", "releases": [ { "tag_to": "HEAD", @@ -5877,11 +5836,9 @@ ], "uri": "git://github.com/openstack/identity-api.git", "module": "identity-api", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "documentation", "releases": [ { "tag_to": "HEAD", @@ -5896,11 +5853,9 @@ ], "uri": "git://github.com/openstack/image-api.git", "module": "image-api", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "documentation", "releases": [ { "tag_to": "HEAD", @@ -5915,11 +5870,9 @@ ], "uri": "git://github.com/openstack/netconn-api.git", "module": "netconn-api", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "documentation", "releases": [ { "tag_to": "HEAD", @@ -5934,11 +5887,9 @@ ], "uri": "git://github.com/openstack/object-api.git", "module": "object-api", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "documentation", "releases": [ { "tag_to": "HEAD", @@ -5953,11 +5904,9 @@ ], "uri": "git://github.com/openstack/volume-api.git", "module": "volume-api", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "documentation", "releases": [ { "branch": "stable/folsom", @@ -5985,11 +5934,9 @@ ], "uri": "git://github.com/openstack/openstack-manuals.git", "module": "openstack-manuals", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "documentation", "releases": [ { "tag_to": "HEAD", @@ -5999,11 +5946,9 @@ ], "uri": "git://github.com/openstack/openstack-doc-tools.git", "module": "openstack-doc-tools", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "documentation", "releases": [ { "tag_to": "HEAD", @@ -6018,11 +5963,9 @@ ], "uri": "git://github.com/openstack/operations-guide.git", "module": "operations-guide", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { - "project_group": "documentation", "releases": [ { "tag_to": "HEAD", @@ -6037,43 +5980,12 @@ ], "uri": "git://github.com/openstack/api-site.git", "module": "api-site", - "organization": "openstack", - "project_type": "openstack" - }, - { - "module": "trove", - "uri": "git://github.com/openstack/trove.git", - "project_type": "openstack", - "organization": "openstack", - "project_group": "incubation" - }, - { - "module": "trove-integration", - "uri": "git://github.com/openstack/trove-integration.git", - "project_type": "openstack", - "organization": "openstack", - "project_group": "incubation" - }, - { - "module": "python-troveclient", - "uri": "git://github.com/openstack/python-troveclient.git", - "project_type": "openstack", - "organization": "openstack", - "project_group": "incubation" - }, - { - "module": "ironic", - "uri": "git://github.com/openstack/ironic.git", - "project_type": "openstack", - "organization": "openstack", - "project_group": "incubation" + "organization": "openstack" }, { "module": "savanna", "uri": "git://github.com/openstack/savanna.git", - "project_type": "openstack", "organization": "openstack", - "project_group": "incubation", "releases": [ { "tag_to": "0.3", @@ -6090,9 +6002,7 @@ { "module": "savanna-dashboard", "uri": "git://github.com/openstack/savanna-dashboard.git", - "project_type": "openstack", "organization": "openstack", - "project_group": "incubation", "releases": [ { "tag_to": "0.3", @@ -6109,9 +6019,7 @@ { "module": "python-savannaclient", "uri": "git://github.com/openstack/python-savannaclient.git", - "project_type": "openstack", "organization": "openstack", - "project_group": "incubation", "releases": [ { "tag_to": "0.3", @@ -6124,58 +6032,44 @@ "tag_from": "0.3" } ] - }, - { - "module": "savanna-extra", - "uri": "git://github.com/openstack/savanna-extra.git", - "project_type": "openstack", - "organization": "openstack", - "project_group": "incubation" - }, - { - "module": "savanna-image-elements", - "uri": "git://github.com/openstack/savanna-image-elements.git", - "project_type": "openstack", - "organization": "openstack", - "project_group": "incubation" - }, - { - "module": "puppet-savanna", - "uri": "git://github.com/stackforge/puppet-savanna.git", - "project_type": "openstack", - "organization": "stackforge", - "project_group": "incubation" } ], "project_sources": [ { "organization": "openstack", - "project_type": "openstack", - "project_group": "other", "exclude": ["openstack"] }, { - "organization": "openstack-dev", - "project_type": "openstack", - "project_group": "other" + "organization": "openstack-dev" }, { - "organization": "openstack-infra", - "project_type": "openstack", - "project_group": "infrastructure" + "organization": "openstack-infra" }, { - "organization": "openstack-ops", - "project_type": "stackforge", - "project_group": null + "organization": "openstack-ops" }, { - "organization": "stackforge", - "project_type": "stackforge", - "project_group": null + "organization": "stackforge" } ], "module_groups": [ + { + "module_group_name": "openstack-core", + "modules": [ + "nova", "python-novaclient", + "neutron", "python-neutronclient", + "keystone", "python-keystoneclient", + "cinder", "python-cinderclient", + "glance", "python-glanceclient", + "swift", "python-swiftclient", + "horizon" + ] + }, + { + "module_group_name": "openstack-docs", + "modules": ["openstack-manuals", "api-site", "operations-guide", "openstack-doc-tools", + "compute-api", "netconn-api", "identity-api", "volume-api", "image-api"] + }, { "module_group_name": "nova-group", "modules": ["nova", "python-novaclient"] @@ -6265,7 +6159,7 @@ }, { "release_name": "Icehouse", - "end_date": "2014-Apr-04" + "end_date": "2014-Apr-17" } ], "mail_lists": ["http://lists.openstack.org/pipermail/openstack-dev/"] diff --git a/etc/default_data.schema.json b/etc/default_data.schema.json index b032d25a6..8738e725e 100644 --- a/etc/default_data.schema.json +++ b/etc/default_data.schema.json @@ -10,7 +10,7 @@ "properties": { "launchpad_id": { "type": "string", - "pattern": "[a-z\\d-]+" + "pattern": "^[a-z\\d\\.-]+$" }, "gerrit_id": { "type": "string" @@ -22,7 +22,7 @@ "type": "array", "items": { "type": "string", - "pattern": "[a-z\\d_\\.-]+@([a-z\\d_\\.-]+\\.)+[a-z]+" + "pattern": "^[a-z\\d_\\.-]+@([a-z\\d\\.-]+\\.)+[a-z]+$" }, "minItems": 1 }, @@ -36,7 +36,7 @@ }, "end_date": { "type": ["string", "null"], - "pattern": "20\\d{2}-\\w{3}-[0-3]\\d" + "pattern": "^20\\d{2}-\\w{3}-[0-3]\\d$" } }, "required": ["company_name", "end_date"], @@ -59,7 +59,7 @@ }, "end_date": { "type": "string", - "pattern": "20\\d{2}-\\w{3}-[0-3]\\d" + "pattern": "^20\\d{2}-\\w{3}-[0-3]\\d$" } }, "required": ["release_name", "end_date"], @@ -74,12 +74,6 @@ "uri": { "type": "string" }, - "project_type": { - "type": "string" - }, - "project_group": { - "type": "string" - }, "organization": { "type": "string" }, @@ -108,7 +102,7 @@ } } }, - "required": ["uri", "project_type", "module", "organization"], + "required": ["uri", "module", "organization"], "additionalProperties": false } }, @@ -124,7 +118,7 @@ "type": "array", "items": { "type": "string", - "pattern": "[a-z\\d\\.]*" + "pattern": "^[a-z\\d\\.-]*$" } } }, @@ -140,11 +134,9 @@ "organization": { "type": "string" }, - "project_type": { - "type": "string" - }, - "project_group": { - "type": ["string", "null"] + "module_group_name": { + "type": "string", + "pattern": "^[\\w-]+$" }, "exclude": { "type": "array", @@ -153,7 +145,7 @@ } } }, - "required": ["organization", "project_type"], + "required": ["organization"], "additionalProperties": false } }, @@ -163,7 +155,8 @@ "type": "object", "properties": { "module_group_name": { - "type": "string" + "type": "string", + "pattern": "^[\\w-]+$" }, "modules": { "type": ["array"], diff --git a/etc/test_default_data.json b/etc/test_default_data.json index b8525f11b..84709a218 100644 --- a/etc/test_default_data.json +++ b/etc/test_default_data.json @@ -55,7 +55,6 @@ "repos": [ { - "project_group": "core", "releases": [ { "tag_to": "2012.1", @@ -88,29 +87,24 @@ ], "uri": "git://github.com/openstack/glance.git", "module": "glance", - "organization": "openstack", - "project_type": "openstack" + "organization": "openstack" }, { "module": "python-glanceclient", - "project_group": "core", - "project_type": "openstack", "organization": "openstack", "uri": "git://github.com/openstack/python-glanceclient.git" }, { "module": "stackalytics", - "project_type": "stackforge", "organization": "stackforge", "uri": "git://github.com/stackforge/stackalytics.git" } ], - "x-project_sources": [ + "project_sources": [ { - "organization": "openstack-dev", - "project_type": "openstack", - "project_group": null + "organization": "openstack-ops", + "module_group_name": "OpenStack-Ops" } ], diff --git a/stackalytics/processor/default_data_processor.py b/stackalytics/processor/default_data_processor.py index 5ad7af614..83b90afe7 100644 --- a/stackalytics/processor/default_data_processor.py +++ b/stackalytics/processor/default_data_processor.py @@ -66,8 +66,6 @@ def _retrieve_project_list_from_github(project_sources): 'branches': ['master'], 'module': repo.name, 'organization': organization, - 'project_type': project_source['project_type'], - 'project_group': project_source['project_group'], 'uri': repo.git_url, 'releases': [] } @@ -76,6 +74,32 @@ def _retrieve_project_list_from_github(project_sources): return repos +def _create_module_groups(project_sources, repos): + organizations = {} + for repo in repos: + ogn = repo['organization'] + if ogn in organizations: + organizations[ogn].append(repo['module']) + else: + organizations[ogn] = [repo['module']] + + ps_organizations = dict([(ps.get('organization'), + ps.get('module_group_name') or + ps.get('organization')) + for ps in project_sources]) + + module_groups = [] + for ogn, modules in organizations.iteritems(): + if ogn in ps_organizations: + module_group_name = ps_organizations[ogn] + else: + module_group_name = ogn + module_groups.append({'module_group_name': module_group_name, + 'modules': modules}) + + return module_groups + + def _update_project_list(default_data): configured_repos = set([r['uri'] for r in default_data['repos']]) @@ -85,6 +109,9 @@ def _update_project_list(default_data): default_data['repos'] += [r for r in repos if r['uri'] not in configured_repos] + default_data['module_groups'] += _create_module_groups( + default_data['project_sources'], default_data['repos']) + def _store_users(runtime_storage_inst, users): for user in users: diff --git a/stackalytics/processor/utils.py b/stackalytics/processor/utils.py index 6883a60fa..67468ddeb 100644 --- a/stackalytics/processor/utils.py +++ b/stackalytics/processor/utils.py @@ -166,3 +166,7 @@ def add_index(sequence, start=1, item_filter=lambda x: True): else: item['index'] = '' return sequence + + +def safe_encode(s): + return urlutils.quote_plus(s.encode('utf-8')) diff --git a/tests/api/test_companies.py b/tests/api/test_companies.py index 271c31595..816576408 100644 --- a/tests/api/test_companies.py +++ b/tests/api/test_companies.py @@ -22,11 +22,9 @@ class TestAPICompanies(test_api.TestAPI): def test_get_companies(self): with test_api.make_runtime_storage( - {'repos': [{'module': 'nova', 'project_type': 'openstack', - 'organization': 'openstack', + {'repos': [{'module': 'nova', 'organization': 'openstack', 'uri': 'git://github.com/openstack/nova.git'}, - {'module': 'glance', 'project_type': 'openstack', - 'organization': 'openstack', + {'module': 'glance', 'organization': 'openstack', 'uri': 'git://github.com/openstack/glance.git'}]}, test_api.make_records(record_type=['commit'], loc=[10, 20, 30], @@ -34,46 +32,49 @@ class TestAPICompanies(test_api.TestAPI): company_name=['NEC', 'IBM', 'NTT']), test_api.make_records(record_type=['review'], primary_key=['0123456789', '9876543210'], + module=['glance'], company_name=['IBM']), test_api.make_records(record_type=['mark'], review_id=['0123456789', '9876543210'], + module=['glance'], company_name=['IBM']), test_api.make_records(record_type=['mark'], review_id=['0123456789'], + module=['glance'], company_name=['NEC'])): - response = self.app.get('/api/1.0/companies?metric=commits') + response = self.app.get('/api/1.0/companies?metric=commits&' + 'module=glance') companies = json.loads(response.data)['companies'] self.assertEqual([{'id': 'ibm', 'text': 'IBM'}, {'id': 'nec', 'text': 'NEC'}, {'id': 'ntt', 'text': 'NTT'}], companies) - response = self.app.get('/api/1.0/companies?metric=marks') + response = self.app.get('/api/1.0/companies?metric=marks&' + 'module=glance') companies = json.loads(response.data)['companies'] self.assertEqual([{'id': 'ibm', 'text': 'IBM'}, {'id': 'nec', 'text': 'NEC'}], companies) response = self.app.get('/api/1.0/companies?metric=commits&' - 'company_name=ib') + 'company_name=ib&module=glance') companies = json.loads(response.data)['companies'] self.assertEqual([{'id': 'ibm', 'text': 'IBM'}], companies) def test_get_company(self): with test_api.make_runtime_storage( - {'repos': [{'module': 'nova', 'project_type': 'openstack', - 'organization': 'openstack', + {'repos': [{'module': 'nova', 'organization': 'openstack', 'uri': 'git://github.com/openstack/nova.git'}, - {'module': 'glance', 'project_type': 'openstack', - 'organization': 'openstack', + {'module': 'glance', 'organization': 'openstack', 'uri': 'git://github.com/openstack/glance.git'}]}, test_api.make_records(record_type=['commit'], loc=[10, 20, 30], module=['glance'], company_name=['NEC', 'IBM', 'NTT'])): - response = self.app.get('/api/1.0/companies/nec') + response = self.app.get('/api/1.0/companies/nec?module=glance') company = json.loads(response.data)['company'] self.assertEqual({'id': 'nec', 'text': 'NEC'}, company) - response = self.app.get('/api/1.0/companies/google') + response = self.app.get('/api/1.0/companies/google?module=glance') self.assertEqual(404, response.status_code) diff --git a/tests/api/test_modules.py b/tests/api/test_modules.py index d8948650e..55f6c4977 100644 --- a/tests/api/test_modules.py +++ b/tests/api/test_modules.py @@ -22,11 +22,9 @@ class TestAPIModules(test_api.TestAPI): def test_get_modules(self): with test_api.make_runtime_storage( - {'repos': [{'module': 'nova', 'project_type': 'openstack', - 'organization': 'openstack', + {'repos': [{'module': 'nova', 'organization': 'openstack', 'uri': 'git://github.com/openstack/nova.git'}, - {'module': 'glance', 'project_type': 'openstack', - 'organization': 'openstack', + {'module': 'glance', 'organization': 'openstack', 'uri': 'git://github.com/openstack/glance.git'}], 'module_groups': [ {'module_group_name': 'nova-group', @@ -37,7 +35,9 @@ class TestAPIModules(test_api.TestAPI): response = self.app.get('/api/1.0/modules') modules = json.loads(response.data)['modules'] self.assertEqual( - [{'id': 'glance', 'modules': ['glance'], 'text': 'glance'}, + [{'group': True, 'id': 'all', 'text': 'All', + 'modules': ['glance', 'nova']}, + {'id': 'glance', 'modules': ['glance'], 'text': 'glance'}, {'id': 'nova', 'modules': ['nova'], 'text': 'nova'}, {'group': True, 'id': 'nova-group', 'text': 'nova-group', 'modules': ['nova', 'python-novaclient']}], modules) @@ -50,8 +50,7 @@ class TestAPIModules(test_api.TestAPI): def test_get_module(self): with test_api.make_runtime_storage( - {'repos': [{'module': 'nova', 'project_type': 'openstack', - 'organization': 'openstack', + {'repos': [{'module': 'nova', 'organization': 'openstack', 'uri': 'git://github.com/openstack/nova.git'}], 'module_groups': [ {'module_group_name': 'nova-group', diff --git a/tests/api/test_stats.py b/tests/api/test_stats.py index 9845d9b1b..d35b6e3bc 100644 --- a/tests/api/test_stats.py +++ b/tests/api/test_stats.py @@ -22,12 +22,13 @@ class TestAPIStats(test_api.TestAPI): def test_get_modules(self): with test_api.make_runtime_storage( - {'repos': [{'module': 'nova', 'project_type': 'openstack', - 'organization': 'openstack', + {'repos': [{'module': 'nova', 'organization': 'openstack', 'uri': 'git://github.com/openstack/nova.git'}, - {'module': 'glance', 'project_type': 'openstack', - 'organization': 'openstack', - 'uri': 'git://github.com/openstack/glance.git'}]}, + {'module': 'glance', 'organization': 'openstack', + 'uri': 'git://github.com/openstack/glance.git'}], + 'module_groups': [ + {'module_group_name': 'openstack', + 'modules': ['nova', 'glance']}]}, test_api.make_records(record_type=['commit'], loc=[10, 20, 30], module=['nova']), @@ -50,6 +51,9 @@ class TestAPIStats(test_api.TestAPI): {'module': 'glance', 'project_type': 'openstack', 'organization': 'openstack', 'uri': 'git://github.com/openstack/glance.git'}], + 'module_groups': [ + {'module_group_name': 'openstack', + 'modules': ['nova', 'glance']}], 'user:john_doe': { 'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe', 'companies': [{'company_name': 'NEC', 'end_date': 0}], @@ -86,6 +90,9 @@ class TestAPIStats(test_api.TestAPI): {'module': 'glance', 'project_type': 'openstack', 'organization': 'openstack', 'uri': 'git://github.com/openstack/glance.git'}], + 'module_groups': [ + {'module_group_name': 'openstack', + 'modules': ['nova', 'glance']}], 'user:john_doe': { 'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe', 'companies': [{'company_name': 'NEC', 'end_date': 0}], diff --git a/tests/api/test_users.py b/tests/api/test_users.py index 11ab530fe..2c4cef58c 100644 --- a/tests/api/test_users.py +++ b/tests/api/test_users.py @@ -27,12 +27,11 @@ class TestAPIUsers(test_api.TestAPI): def test_users(self): with test_api.make_runtime_storage( - {'repos': [{'module': 'nova', 'project_type': 'openstack', - 'organization': 'openstack', + {'repos': [{'module': 'nova', 'organization': 'openstack', 'uri': 'git://github.com/openstack/nova.git'}]}, - test_api.make_records(record_type=['commit'], + test_api.make_records(record_type=['commit'], module=['nova'], user_id=['john_doe', 'bill_smith'])): - response = self.app.get('/api/1.0/users') + response = self.app.get('/api/1.0/users?module=nova') users = json.loads(response.data)['users'] self.assertEqual(2, len(users)) self.assertIn({'id': 'john_doe', 'text': 'John Doe'}, users) @@ -40,12 +39,11 @@ class TestAPIUsers(test_api.TestAPI): def test_users_search(self): with test_api.make_runtime_storage( - {'repos': [{'module': 'nova', 'project_type': 'openstack', - 'organization': 'openstack', + {'repos': [{'module': 'nova', 'organization': 'openstack', 'uri': 'git://github.com/openstack/nova.git'}]}, - test_api.make_records(record_type=['commit'], + test_api.make_records(record_type=['commit'], module=['nova'], user_name=['John Doe', 'Bill Smith'])): - response = self.app.get('/api/1.0/users?query=doe') + response = self.app.get('/api/1.0/users?module=nova&query=doe') users = json.loads(response.data)['users'] self.assertEqual(1, len(users)) self.assertIn({'id': 'john_doe', 'text': 'John Doe'}, users) @@ -55,7 +53,9 @@ class TestAPIUsers(test_api.TestAPI): {'user:john_doe': { 'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe', 'companies': [{'company_name': 'NEC', 'end_date': 0}], - 'emails': 'john_doe@gmail.com'}}): + 'emails': 'john_doe@gmail.com'}}, + test_api.make_records(record_type=['commit'], module=['nova'], + user_name=['John Doe', 'Bill Smith'])): response = self.app.get('/api/1.0/users/john_doe') user = json.loads(response.data)['user'] self.assertEqual('john_doe', user['user_id']) @@ -65,6 +65,12 @@ class TestAPIUsers(test_api.TestAPI): {'user:john_doe': { 'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe', 'companies': [{'company_name': 'NEC', 'end_date': 0}], - 'emails': 'john_doe@gmail.com'}}): + 'emails': 'john_doe@gmail.com'}, + 'repos': [{'module': 'nova', 'organization': 'openstack', + 'uri': 'git://github.com/openstack/nova.git'}], + 'module_groups': [ + {'module_group_name': 'openstack', 'modules': ['nova']}]}, + test_api.make_records(record_type=['commit'], module=['nova'], + user_name=['John Doe', 'Bill Smith'])): response = self.app.get('/api/1.0/users/nonexistent') self.assertEqual(404, response.status_code) diff --git a/tests/unit/test_default_data_processor.py b/tests/unit/test_default_data_processor.py index e7eea4a11..9e72e3a9f 100644 --- a/tests/unit/test_default_data_processor.py +++ b/tests/unit/test_default_data_processor.py @@ -56,18 +56,32 @@ class TestDefaultDataProcessor(testtools.TestCase): with mock.patch('stackalytics.processor.default_data_processor.' '_retrieve_project_list_from_github') as retriever: retriever.return_value = [ - {'module': 'nova', 'uri': 'git://github.com/openstack/nova'}, - {'module': 'qa', 'uri': 'git://github.com/openstack/qa'}, + {'module': 'nova', 'uri': 'git://github.com/openstack/nova', + 'organization': 'openstack'}, + {'module': 'qa', 'uri': 'git://github.com/openstack/qa', + 'organization': 'openstack'}, ] dd = { 'repos': [ - {'module': 'qa', 'uri': 'git://github.com/openstack/qa'}, + {'module': 'qa', 'uri': 'git://github.com/openstack/qa', + 'organization': 'openstack'}, + {'module': 'tux', 'uri': 'git://github.com/stackforge/tux', + 'organization': 'stackforge'}, ], - 'project_sources': ['any'] + 'project_sources': [{'organization': 'openstack', + 'module_group_name': 'OpenStack'}], + 'module_groups': [], } default_data_processor._update_project_list(dd) - self.assertEqual(2, len(dd['repos'])) + self.assertEqual(3, len(dd['repos'])) self.assertIn('qa', set([r['module'] for r in dd['repos']])) self.assertIn('nova', set([r['module'] for r in dd['repos']])) + self.assertIn('tux', set([r['module'] for r in dd['repos']])) + + self.assertEqual(2, len(dd['module_groups'])) + self.assertIn({'module_group_name': 'OpenStack', + 'modules': ['qa', 'nova']}, dd['module_groups']) + self.assertIn({'module_group_name': 'stackforge', + 'modules': ['tux']}, dd['module_groups'])