diff --git a/dashboard/decorators.py b/dashboard/decorators.py
index f301ef5d9..c012a21e9 100644
--- a/dashboard/decorators.py
+++ b/dashboard/decorators.py
@@ -58,7 +58,6 @@ def record_filter(ignore=None, use_default=True):
@functools.wraps(f)
def record_filter_decorated_function(*args, **kwargs):
- vault_inst = vault.get_vault()
memory_storage_inst = vault.get_memory_storage()
record_ids = set(memory_storage_inst.get_record_ids()) # a copy
@@ -70,19 +69,6 @@ def record_filter(ignore=None, use_default=True):
memory_storage_inst.get_record_ids_by_modules(
vault.resolve_modules(param)))
- if 'project_type' not in ignore:
- param = parameters.get_parameter(kwargs, 'project_type',
- 'project_types', use_default)
- if param:
- ptgi = vault_inst['project_type_group_index']
- modules = set()
- for project_type in param:
- project_type = project_type.lower()
- if project_type in ptgi:
- modules |= ptgi[project_type]
- record_ids &= (
- memory_storage_inst.get_record_ids_by_modules(modules))
-
if 'user_id' not in ignore:
param = parameters.get_parameter(kwargs, 'user_id', 'user_ids')
param = [u for u in param
@@ -265,11 +251,6 @@ def templated(template=None, return_code=200):
ctx['metric'] = metric or parameters.get_default('metric')
ctx['metric_label'] = parameters.METRIC_LABELS[ctx['metric']]
- project_type = flask.request.args.get('project_type')
- if not vault.is_project_type_valid(project_type):
- project_type = parameters.get_default('project_type')
- ctx['project_type'] = project_type
-
release = flask.request.args.get('release')
releases = vault_inst['releases']
if release:
@@ -284,7 +265,6 @@ def templated(template=None, return_code=200):
ctx['review_nth'] = (flask.request.args.get('review_nth') or
parameters.get_default('review_nth'))
- ctx['project_type_options'] = vault.get_project_type_options()
ctx['release_options'] = vault.get_release_options()
ctx['metric_options'] = sorted(parameters.METRIC_LABELS.items(),
key=lambda x: x[0])
@@ -293,7 +273,11 @@ def templated(template=None, return_code=200):
ctx['company_original'] = (
vault.get_memory_storage().get_original_company_name(
ctx['company']))
- ctx['module'] = parameters.get_single_parameter(kwargs, 'module')
+
+ module = parameters.get_single_parameter(kwargs, 'module')
+ ctx['module'] = module
+ ctx['module_inst'] = vault_inst['module_id_index'][module]
+
ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id')
ctx['page_title'] = helpers.make_page_title(
ctx['company'], ctx['user_id'], ctx['module'], ctx['release'])
diff --git a/dashboard/helpers.py b/dashboard/helpers.py
index d8fc59c88..462f602b7 100644
--- a/dashboard/helpers.py
+++ b/dashboard/helpers.py
@@ -21,7 +21,6 @@ import six
from dashboard import parameters
from dashboard import vault
-from stackalytics.openstack.common.py3kcompat import urlutils
from stackalytics.processor import utils
@@ -178,13 +177,8 @@ def format_launchpad_module_link(module):
return '%s' % (module, module)
-def safe_encode(s):
- return urlutils.quote_plus(s.encode('utf-8'))
-
-
def make_link(title, uri=None, options=None):
- param_names = ('release', 'project_type', 'module', 'company', 'user_id',
- 'metric')
+ param_names = ('release', 'module', 'company', 'user_id', 'metric')
param_values = {}
for param_name in param_names:
v = parameters.get_parameter({}, param_name, param_name)
@@ -193,7 +187,7 @@ def make_link(title, uri=None, options=None):
if options:
param_values.update(options)
if param_values:
- uri += '?' + '&'.join(['%s=%s' % (n, safe_encode(v))
+ uri += '?' + '&'.join(['%s=%s' % (n, utils.safe_encode(v))
for n, v in six.iteritems(param_values)])
return '%(title)s' % {'uri': uri, 'title': title}
diff --git a/dashboard/parameters.py b/dashboard/parameters.py
index 85fafce35..8a6091e57 100644
--- a/dashboard/parameters.py
+++ b/dashboard/parameters.py
@@ -25,7 +25,7 @@ LOG = logging.getLogger(__name__)
DEFAULTS = {
'metric': 'commits',
'release': 'icehouse',
- 'project_type': 'openstack',
+ 'module': 'openstack',
'review_nth': 5,
}
diff --git a/dashboard/static/js/stackalytics-ui.js b/dashboard/static/js/stackalytics-ui.js
index f4ec09975..3d56b1951 100644
--- a/dashboard/static/js/stackalytics-ui.js
+++ b/dashboard/static/js/stackalytics-ui.js
@@ -276,7 +276,6 @@ function make_std_options() {
var options = {};
options['release'] = $('#release').val();
options['metric'] = $('#metric').val();
- options['project_type'] = $('#project_type').val();
options['module'] = $('#module').val() || '';
options['company'] = $('#company').val() || '';
options['user_id'] = $('#user').val() || '';
@@ -436,7 +435,7 @@ function init_selectors(base_url) {
});
$("#module").select2({
- allowClear: true,
+ allowClear: false,
ajax: {
url: make_uri(base_url + "/api/1.0/modules"),
dataType: 'jsonp',
diff --git a/dashboard/templates/kpi/example.html b/dashboard/templates/kpi/example.html
index 41160004f..6166e90db 100644
--- a/dashboard/templates/kpi/example.html
+++ b/dashboard/templates/kpi/example.html
@@ -8,18 +8,18 @@
diff --git a/dashboard/templates/layout.html b/dashboard/templates/layout.html
index c4e45850b..48f252b56 100644
--- a/dashboard/templates/layout.html
+++ b/dashboard/templates/layout.html
@@ -59,7 +59,7 @@
About ↗
@@ -67,32 +67,27 @@
-
-
-
-
-
-
+
-
+
-
+
-
+
-
+
diff --git a/dashboard/templates/overview.html b/dashboard/templates/overview.html
index e535c08a8..c7c9e6187 100644
--- a/dashboard/templates/overview.html
+++ b/dashboard/templates/overview.html
@@ -4,9 +4,9 @@
{% set show_company_breakdown = (not company) and (not user_id) %}
{% set show_engineer_breakdown = (not user_id) %}
{% set show_bp_breakdown = (metric in ['bpd', 'bpc']) %}
-{% set show_module_breakdown = (not module) %}
+{% set show_module_breakdown = (not module) or (module_inst['group']) %}
{% set show_user_activity = (user_id) %}
-{% set show_module_activity = (module) and (not user_id) %}
+{% set show_module_activity = (module) and (not user_id) and (not module_inst['group']) %}
{% set show_activity = (show_user_activity) or (show_module_activity) %}
{% set show_user_contribution = (user_id) or (company) %}
{% set show_module_contribution = (module) and (not user_id) %}
diff --git a/dashboard/templates/reports/contribution.html b/dashboard/templates/reports/contribution.html
index 008e10d62..dd7e6cefa 100644
--- a/dashboard/templates/reports/contribution.html
+++ b/dashboard/templates/reports/contribution.html
@@ -12,7 +12,7 @@ Contribution into {{ module }} for the last {{ days }} days
var table_id = "review_stats_table";
$.ajax({
- url: make_uri("/api/1.0/stats/engineers_extended?project_type=all&module={{ module }}&release=all&start_date={{ start_date }}"),
+ url: make_uri("/api/1.0/stats/engineers_extended?metric=marks&module={{ module }}&release=all&start_date={{ start_date }}"),
dataType: "json",
success: function (data) {
var tableData = data["stats"];
diff --git a/dashboard/templates/widget.html b/dashboard/templates/widget.html
index e9a178999..c45688ddd 100644
--- a/dashboard/templates/widget.html
+++ b/dashboard/templates/widget.html
@@ -121,8 +121,8 @@
-
-
+
+
diff --git a/dashboard/vault.py b/dashboard/vault.py
index 7edd076c0..0858609a2 100644
--- a/dashboard/vault.py
+++ b/dashboard/vault.py
@@ -17,7 +17,6 @@ import os
import flask
from oslo.config import cfg
-import six
from dashboard import memory_storage
from stackalytics.openstack.common import log as logging
@@ -39,7 +38,6 @@ def get_vault():
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED)
- init_project_types(vault)
init_releases(vault)
flask.current_app.stackalytics_vault = vault
@@ -55,7 +53,6 @@ def get_vault():
vault['runtime_storage'].get_update(os.getpid()))
if have_updates:
- init_project_types(vault)
init_releases(vault)
init_module_groups(vault)
@@ -84,53 +81,20 @@ def init_releases(vault):
vault['releases'] = releases_map
-def init_project_types(vault):
- runtime_storage_inst = vault['runtime_storage']
- project_type_options = {}
- project_type_group_index = {'all': set(['unknown'])}
-
- for repo in utils.load_repos(runtime_storage_inst):
- project_type = repo['project_type'].lower()
- project_group = None
- if ('project_group' in repo) and (repo['project_group']):
- project_group = repo['project_group'].lower()
-
- if project_type in project_type_options:
- if project_group:
- project_type_options[project_type].add(project_group)
- else:
- if project_group:
- project_type_options[project_type] = set([project_group])
- else:
- project_type_options[project_type] = set()
-
- module = repo['module']
- if project_type in project_type_group_index:
- project_type_group_index[project_type].add(module)
- else:
- project_type_group_index[project_type] = set([module])
-
- if project_group:
- if project_group in project_type_group_index:
- project_type_group_index[project_group].add(module)
- else:
- project_type_group_index[project_group] = set([module])
-
- project_type_group_index['all'].add(module)
-
- vault['project_type_options'] = project_type_options
- vault['project_type_group_index'] = project_type_group_index
-
-
def init_module_groups(vault):
runtime_storage_inst = vault['runtime_storage']
+ memory_storage_inst = vault['memory_storage']
+
module_index = {}
module_id_index = {}
module_groups = runtime_storage_inst.get_by_key('module_groups') or []
+ module_groups.append({'module_group_name': 'All',
+ 'modules': set(memory_storage_inst.get_modules())})
+
for module_group in module_groups:
module_group_name = module_group['module_group_name']
- module_group_id = module_group_name.lower()
+ module_group_id = utils.safe_encode(module_group_name.lower())
module_id_index[module_group_id] = {
'group': True,
@@ -146,10 +110,9 @@ def init_module_groups(vault):
else:
module_index[module] = set([module_group_id])
- memory_storage_inst = vault['memory_storage']
for module in memory_storage_inst.get_modules():
module_id_index[module] = {
- 'id': module.lower(),
+ 'id': utils.safe_encode(module.lower()),
'text': module,
'modules': [module.lower()],
}
@@ -159,10 +122,6 @@ def init_module_groups(vault):
vault['module_groups'] = module_groups
-def get_project_type_options():
- return get_vault()['project_type_options']
-
-
def get_release_options():
runtime_storage_inst = get_vault()['runtime_storage']
releases = (runtime_storage_inst.get_by_key('releases') or [None])[1:]
@@ -171,28 +130,13 @@ def get_release_options():
return releases
-def is_project_type_valid(project_type):
- if not project_type:
- return False
- project_type = project_type.lower()
- if project_type == 'all':
- return True
- project_types = get_project_type_options()
- if project_type in project_types:
- return True
- for p, g in six.iteritems(project_types):
- if project_type in g:
- return True
- return False
-
-
def get_user_from_runtime_storage(user_id):
runtime_storage_inst = get_vault()['runtime_storage']
return utils.load_user(runtime_storage_inst, user_id)
def resolve_modules(module_ids):
- module_id_index = get_vault()['module_id_index']
+ module_id_index = get_vault().get('module_id_index') or {}
modules = set()
for module_id in module_ids:
if module_id in module_id_index:
diff --git a/dashboard/web.py b/dashboard/web.py
index 2da2d8771..3aaf4e336 100644
--- a/dashboard/web.py
+++ b/dashboard/web.py
@@ -242,7 +242,7 @@ def get_companies_json(records):
continue
if name.lower().find(query.lower()) >= 0:
options.add(name)
- result = [{'id': helpers.safe_encode(c.lower()), 'text': c}
+ result = [{'id': utils.safe_encode(c.lower()), 'text': c}
for c in sorted(options)]
return result
@@ -406,7 +406,7 @@ def get_metric_json(metric):
@decorators.exception_handler()
def get_project_types_json():
return [{'id': m, 'text': m, 'items': list(t)}
- for m, t in six.iteritems(vault.get_project_type_options())]
+ for m, t in vault.get_project_type_options().iteritems()]
@app.route('/api/1.0/project_types/
')
@@ -414,7 +414,7 @@ def get_project_types_json():
@decorators.exception_handler()
def get_project_type_json(project_type):
if project_type != 'all':
- for pt, groups in six.iteritems(vault.get_project_type_options()):
+ for pt, groups in vault.get_project_type_options().iteritems():
if (project_type == pt) or (project_type in groups):
break
else:
diff --git a/etc/default_data.json b/etc/default_data.json
index dd2c173f1..ddff3b448 100644
--- a/etc/default_data.json
+++ b/etc/default_data.json
@@ -323,7 +323,7 @@
}
],
"user_name": "armando-migliaccio",
- "emails": ["amigliaccio@nicira.com", "Armando.Migliaccio@eu.citrix.com", "amigliaccio@internap.com", "armamig@gmail.com", "armando.migliaccio@citrix.com", "armando.migliaccio@eu.citrix.com"]
+ "emails": ["amigliaccio@nicira.com", "armando.migliaccio@eu.citrix.com", "amigliaccio@internap.com", "armamig@gmail.com", "armando.migliaccio@citrix.com"]
},
{
"launchpad_id": "arnaudleg",
@@ -2303,7 +2303,7 @@
}
],
"user_name": "Juan Perez",
- "emails": ["Juan_Perez@cable.comcast.com"]
+ "emails": ["juan_perez@cable.comcast.com"]
},
{
"launchpad_id": "jun-park-earth",
@@ -3104,7 +3104,7 @@
}
],
"user_name": "Zhenguo Niu",
- "emails": ["zhenguo@unitedstack.com", "Niu.ZGlinux@gmail.com"]
+ "emails": ["zhenguo@unitedstack.com", "niu.zglinux@gmail.com"]
},
{
"launchpad_id": "nobodycam",
@@ -3692,7 +3692,7 @@
}
],
"user_name": "Sean Gossard",
- "emails": ["sgossard@parelastic.com", "sgossard@tesora.com", "seanGossard@usa.com"]
+ "emails": ["sgossard@parelastic.com", "sgossard@tesora.com", "seangossard@usa.com"]
},
{
"launchpad_id": "sean-mccully",
@@ -4292,7 +4292,7 @@
}
],
"user_name": "Trevor R Jr",
- "emails": ["vmtrooper@gmail.com", "Trevor.RobertsJr@vce.com"]
+ "emails": ["vmtrooper@gmail.com", "trevor.robertsjr@vce.com"]
},
{
"launchpad_id": "vsharshov",
@@ -5295,7 +5295,6 @@
],
"repos": [
{
- "project_group": "core",
"releases": [
{
"tag_to": "2012.1",
@@ -5328,11 +5327,9 @@
],
"uri": "git://github.com/openstack/nova.git",
"module": "nova",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "2012.1",
@@ -5365,11 +5362,9 @@
],
"uri": "git://github.com/openstack/keystone.git",
"module": "keystone",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"branch": "stable/folsom",
@@ -5397,11 +5392,9 @@
],
"uri": "git://github.com/openstack/cinder.git",
"module": "cinder",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "2012.1",
@@ -5434,11 +5427,9 @@
],
"uri": "git://github.com/openstack/glance.git",
"module": "glance",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "2012.1",
@@ -5471,11 +5462,9 @@
],
"uri": "git://github.com/openstack/neutron.git",
"module": "neutron",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "2012.1",
@@ -5508,11 +5497,9 @@
],
"uri": "git://github.com/openstack/horizon.git",
"module": "horizon",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "1.4.8",
@@ -5545,11 +5532,9 @@
],
"uri": "git://github.com/openstack/swift.git",
"module": "swift",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "0.2.0",
@@ -5574,11 +5559,9 @@
],
"uri": "git://github.com/openstack/python-keystoneclient.git",
"module": "python-keystoneclient",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "2.9.0",
@@ -5603,11 +5586,9 @@
],
"uri": "git://github.com/openstack/python-novaclient.git",
"module": "python-novaclient",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "1.0.3",
@@ -5627,11 +5608,9 @@
],
"uri": "git://github.com/openstack/python-cinderclient.git",
"module": "python-cinderclient",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "0.9.0",
@@ -5651,11 +5630,9 @@
],
"uri": "git://github.com/openstack/python-glanceclient.git",
"module": "python-glanceclient",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "2.1",
@@ -5680,11 +5657,9 @@
],
"uri": "git://github.com/openstack/python-neutronclient.git",
"module": "python-neutronclient",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "1.3.0",
@@ -5704,11 +5679,9 @@
],
"uri": "git://github.com/openstack/python-swiftclient.git",
"module": "python-swiftclient",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"branch": "stable/grizzly",
@@ -5730,11 +5703,9 @@
],
"uri": "git://github.com/openstack/heat.git",
"module": "heat",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "0.2.2",
@@ -5754,11 +5725,9 @@
],
"uri": "git://github.com/openstack/python-heatclient.git",
"module": "python-heatclient",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"branch": "stable/grizzly",
@@ -5780,11 +5749,9 @@
],
"uri": "git://github.com/openstack/ceilometer.git",
"module": "ceilometer",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"tag_to": "1.0.6",
@@ -5799,11 +5766,9 @@
],
"uri": "git://github.com/openstack/python-ceilometerclient.git",
"module": "python-ceilometerclient",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"branch": "stable/havana",
@@ -5819,11 +5784,9 @@
],
"uri": "git://github.com/openstack/oslo-incubator.git",
"module": "oslo-incubator",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "core",
"releases": [
{
"branch": "stable/havana",
@@ -5839,11 +5802,9 @@
],
"uri": "git://github.com/openstack/oslo.config.git",
"module": "oslo.config",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "documentation",
"releases": [
{
"tag_to": "HEAD",
@@ -5858,11 +5819,9 @@
],
"uri": "git://github.com/openstack/compute-api.git",
"module": "compute-api",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "documentation",
"releases": [
{
"tag_to": "HEAD",
@@ -5877,11 +5836,9 @@
],
"uri": "git://github.com/openstack/identity-api.git",
"module": "identity-api",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "documentation",
"releases": [
{
"tag_to": "HEAD",
@@ -5896,11 +5853,9 @@
],
"uri": "git://github.com/openstack/image-api.git",
"module": "image-api",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "documentation",
"releases": [
{
"tag_to": "HEAD",
@@ -5915,11 +5870,9 @@
],
"uri": "git://github.com/openstack/netconn-api.git",
"module": "netconn-api",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "documentation",
"releases": [
{
"tag_to": "HEAD",
@@ -5934,11 +5887,9 @@
],
"uri": "git://github.com/openstack/object-api.git",
"module": "object-api",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "documentation",
"releases": [
{
"tag_to": "HEAD",
@@ -5953,11 +5904,9 @@
],
"uri": "git://github.com/openstack/volume-api.git",
"module": "volume-api",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "documentation",
"releases": [
{
"branch": "stable/folsom",
@@ -5985,11 +5934,9 @@
],
"uri": "git://github.com/openstack/openstack-manuals.git",
"module": "openstack-manuals",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "documentation",
"releases": [
{
"tag_to": "HEAD",
@@ -5999,11 +5946,9 @@
],
"uri": "git://github.com/openstack/openstack-doc-tools.git",
"module": "openstack-doc-tools",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "documentation",
"releases": [
{
"tag_to": "HEAD",
@@ -6018,11 +5963,9 @@
],
"uri": "git://github.com/openstack/operations-guide.git",
"module": "operations-guide",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
- "project_group": "documentation",
"releases": [
{
"tag_to": "HEAD",
@@ -6037,43 +5980,12 @@
],
"uri": "git://github.com/openstack/api-site.git",
"module": "api-site",
- "organization": "openstack",
- "project_type": "openstack"
- },
- {
- "module": "trove",
- "uri": "git://github.com/openstack/trove.git",
- "project_type": "openstack",
- "organization": "openstack",
- "project_group": "incubation"
- },
- {
- "module": "trove-integration",
- "uri": "git://github.com/openstack/trove-integration.git",
- "project_type": "openstack",
- "organization": "openstack",
- "project_group": "incubation"
- },
- {
- "module": "python-troveclient",
- "uri": "git://github.com/openstack/python-troveclient.git",
- "project_type": "openstack",
- "organization": "openstack",
- "project_group": "incubation"
- },
- {
- "module": "ironic",
- "uri": "git://github.com/openstack/ironic.git",
- "project_type": "openstack",
- "organization": "openstack",
- "project_group": "incubation"
+ "organization": "openstack"
},
{
"module": "savanna",
"uri": "git://github.com/openstack/savanna.git",
- "project_type": "openstack",
"organization": "openstack",
- "project_group": "incubation",
"releases": [
{
"tag_to": "0.3",
@@ -6090,9 +6002,7 @@
{
"module": "savanna-dashboard",
"uri": "git://github.com/openstack/savanna-dashboard.git",
- "project_type": "openstack",
"organization": "openstack",
- "project_group": "incubation",
"releases": [
{
"tag_to": "0.3",
@@ -6109,9 +6019,7 @@
{
"module": "python-savannaclient",
"uri": "git://github.com/openstack/python-savannaclient.git",
- "project_type": "openstack",
"organization": "openstack",
- "project_group": "incubation",
"releases": [
{
"tag_to": "0.3",
@@ -6124,58 +6032,44 @@
"tag_from": "0.3"
}
]
- },
- {
- "module": "savanna-extra",
- "uri": "git://github.com/openstack/savanna-extra.git",
- "project_type": "openstack",
- "organization": "openstack",
- "project_group": "incubation"
- },
- {
- "module": "savanna-image-elements",
- "uri": "git://github.com/openstack/savanna-image-elements.git",
- "project_type": "openstack",
- "organization": "openstack",
- "project_group": "incubation"
- },
- {
- "module": "puppet-savanna",
- "uri": "git://github.com/stackforge/puppet-savanna.git",
- "project_type": "openstack",
- "organization": "stackforge",
- "project_group": "incubation"
}
],
"project_sources": [
{
"organization": "openstack",
- "project_type": "openstack",
- "project_group": "other",
"exclude": ["openstack"]
},
{
- "organization": "openstack-dev",
- "project_type": "openstack",
- "project_group": "other"
+ "organization": "openstack-dev"
},
{
- "organization": "openstack-infra",
- "project_type": "openstack",
- "project_group": "infrastructure"
+ "organization": "openstack-infra"
},
{
- "organization": "openstack-ops",
- "project_type": "stackforge",
- "project_group": null
+ "organization": "openstack-ops"
},
{
- "organization": "stackforge",
- "project_type": "stackforge",
- "project_group": null
+ "organization": "stackforge"
}
],
"module_groups": [
+ {
+ "module_group_name": "openstack-core",
+ "modules": [
+ "nova", "python-novaclient",
+ "neutron", "python-neutronclient",
+ "keystone", "python-keystoneclient",
+ "cinder", "python-cinderclient",
+ "glance", "python-glanceclient",
+ "swift", "python-swiftclient",
+ "horizon"
+ ]
+ },
+ {
+ "module_group_name": "openstack-docs",
+ "modules": ["openstack-manuals", "api-site", "operations-guide", "openstack-doc-tools",
+ "compute-api", "netconn-api", "identity-api", "volume-api", "image-api"]
+ },
{
"module_group_name": "nova-group",
"modules": ["nova", "python-novaclient"]
@@ -6265,7 +6159,7 @@
},
{
"release_name": "Icehouse",
- "end_date": "2014-Apr-04"
+ "end_date": "2014-Apr-17"
}
],
"mail_lists": ["http://lists.openstack.org/pipermail/openstack-dev/"]
diff --git a/etc/default_data.schema.json b/etc/default_data.schema.json
index b032d25a6..8738e725e 100644
--- a/etc/default_data.schema.json
+++ b/etc/default_data.schema.json
@@ -10,7 +10,7 @@
"properties": {
"launchpad_id": {
"type": "string",
- "pattern": "[a-z\\d-]+"
+ "pattern": "^[a-z\\d\\.-]+$"
},
"gerrit_id": {
"type": "string"
@@ -22,7 +22,7 @@
"type": "array",
"items": {
"type": "string",
- "pattern": "[a-z\\d_\\.-]+@([a-z\\d_\\.-]+\\.)+[a-z]+"
+ "pattern": "^[a-z\\d_\\.-]+@([a-z\\d\\.-]+\\.)+[a-z]+$"
},
"minItems": 1
},
@@ -36,7 +36,7 @@
},
"end_date": {
"type": ["string", "null"],
- "pattern": "20\\d{2}-\\w{3}-[0-3]\\d"
+ "pattern": "^20\\d{2}-\\w{3}-[0-3]\\d$"
}
},
"required": ["company_name", "end_date"],
@@ -59,7 +59,7 @@
},
"end_date": {
"type": "string",
- "pattern": "20\\d{2}-\\w{3}-[0-3]\\d"
+ "pattern": "^20\\d{2}-\\w{3}-[0-3]\\d$"
}
},
"required": ["release_name", "end_date"],
@@ -74,12 +74,6 @@
"uri": {
"type": "string"
},
- "project_type": {
- "type": "string"
- },
- "project_group": {
- "type": "string"
- },
"organization": {
"type": "string"
},
@@ -108,7 +102,7 @@
}
}
},
- "required": ["uri", "project_type", "module", "organization"],
+ "required": ["uri", "module", "organization"],
"additionalProperties": false
}
},
@@ -124,7 +118,7 @@
"type": "array",
"items": {
"type": "string",
- "pattern": "[a-z\\d\\.]*"
+ "pattern": "^[a-z\\d\\.-]*$"
}
}
},
@@ -140,11 +134,9 @@
"organization": {
"type": "string"
},
- "project_type": {
- "type": "string"
- },
- "project_group": {
- "type": ["string", "null"]
+ "module_group_name": {
+ "type": "string",
+ "pattern": "^[\\w-]+$"
},
"exclude": {
"type": "array",
@@ -153,7 +145,7 @@
}
}
},
- "required": ["organization", "project_type"],
+ "required": ["organization"],
"additionalProperties": false
}
},
@@ -163,7 +155,8 @@
"type": "object",
"properties": {
"module_group_name": {
- "type": "string"
+ "type": "string",
+ "pattern": "^[\\w-]+$"
},
"modules": {
"type": ["array"],
diff --git a/etc/test_default_data.json b/etc/test_default_data.json
index b8525f11b..84709a218 100644
--- a/etc/test_default_data.json
+++ b/etc/test_default_data.json
@@ -55,7 +55,6 @@
"repos": [
{
- "project_group": "core",
"releases": [
{
"tag_to": "2012.1",
@@ -88,29 +87,24 @@
],
"uri": "git://github.com/openstack/glance.git",
"module": "glance",
- "organization": "openstack",
- "project_type": "openstack"
+ "organization": "openstack"
},
{
"module": "python-glanceclient",
- "project_group": "core",
- "project_type": "openstack",
"organization": "openstack",
"uri": "git://github.com/openstack/python-glanceclient.git"
},
{
"module": "stackalytics",
- "project_type": "stackforge",
"organization": "stackforge",
"uri": "git://github.com/stackforge/stackalytics.git"
}
],
- "x-project_sources": [
+ "project_sources": [
{
- "organization": "openstack-dev",
- "project_type": "openstack",
- "project_group": null
+ "organization": "openstack-ops",
+ "module_group_name": "OpenStack-Ops"
}
],
diff --git a/stackalytics/processor/default_data_processor.py b/stackalytics/processor/default_data_processor.py
index 5ad7af614..83b90afe7 100644
--- a/stackalytics/processor/default_data_processor.py
+++ b/stackalytics/processor/default_data_processor.py
@@ -66,8 +66,6 @@ def _retrieve_project_list_from_github(project_sources):
'branches': ['master'],
'module': repo.name,
'organization': organization,
- 'project_type': project_source['project_type'],
- 'project_group': project_source['project_group'],
'uri': repo.git_url,
'releases': []
}
@@ -76,6 +74,32 @@ def _retrieve_project_list_from_github(project_sources):
return repos
+def _create_module_groups(project_sources, repos):
+ organizations = {}
+ for repo in repos:
+ ogn = repo['organization']
+ if ogn in organizations:
+ organizations[ogn].append(repo['module'])
+ else:
+ organizations[ogn] = [repo['module']]
+
+ ps_organizations = dict([(ps.get('organization'),
+ ps.get('module_group_name') or
+ ps.get('organization'))
+ for ps in project_sources])
+
+ module_groups = []
+ for ogn, modules in organizations.iteritems():
+ if ogn in ps_organizations:
+ module_group_name = ps_organizations[ogn]
+ else:
+ module_group_name = ogn
+ module_groups.append({'module_group_name': module_group_name,
+ 'modules': modules})
+
+ return module_groups
+
+
def _update_project_list(default_data):
configured_repos = set([r['uri'] for r in default_data['repos']])
@@ -85,6 +109,9 @@ def _update_project_list(default_data):
default_data['repos'] += [r for r in repos
if r['uri'] not in configured_repos]
+ default_data['module_groups'] += _create_module_groups(
+ default_data['project_sources'], default_data['repos'])
+
def _store_users(runtime_storage_inst, users):
for user in users:
diff --git a/stackalytics/processor/utils.py b/stackalytics/processor/utils.py
index 6883a60fa..67468ddeb 100644
--- a/stackalytics/processor/utils.py
+++ b/stackalytics/processor/utils.py
@@ -166,3 +166,7 @@ def add_index(sequence, start=1, item_filter=lambda x: True):
else:
item['index'] = ''
return sequence
+
+
+def safe_encode(s):
+ return urlutils.quote_plus(s.encode('utf-8'))
diff --git a/tests/api/test_companies.py b/tests/api/test_companies.py
index 271c31595..816576408 100644
--- a/tests/api/test_companies.py
+++ b/tests/api/test_companies.py
@@ -22,11 +22,9 @@ class TestAPICompanies(test_api.TestAPI):
def test_get_companies(self):
with test_api.make_runtime_storage(
- {'repos': [{'module': 'nova', 'project_type': 'openstack',
- 'organization': 'openstack',
+ {'repos': [{'module': 'nova', 'organization': 'openstack',
'uri': 'git://github.com/openstack/nova.git'},
- {'module': 'glance', 'project_type': 'openstack',
- 'organization': 'openstack',
+ {'module': 'glance', 'organization': 'openstack',
'uri': 'git://github.com/openstack/glance.git'}]},
test_api.make_records(record_type=['commit'],
loc=[10, 20, 30],
@@ -34,46 +32,49 @@ class TestAPICompanies(test_api.TestAPI):
company_name=['NEC', 'IBM', 'NTT']),
test_api.make_records(record_type=['review'],
primary_key=['0123456789', '9876543210'],
+ module=['glance'],
company_name=['IBM']),
test_api.make_records(record_type=['mark'],
review_id=['0123456789', '9876543210'],
+ module=['glance'],
company_name=['IBM']),
test_api.make_records(record_type=['mark'],
review_id=['0123456789'],
+ module=['glance'],
company_name=['NEC'])):
- response = self.app.get('/api/1.0/companies?metric=commits')
+ response = self.app.get('/api/1.0/companies?metric=commits&'
+ 'module=glance')
companies = json.loads(response.data)['companies']
self.assertEqual([{'id': 'ibm', 'text': 'IBM'},
{'id': 'nec', 'text': 'NEC'},
{'id': 'ntt', 'text': 'NTT'}], companies)
- response = self.app.get('/api/1.0/companies?metric=marks')
+ response = self.app.get('/api/1.0/companies?metric=marks&'
+ 'module=glance')
companies = json.loads(response.data)['companies']
self.assertEqual([{'id': 'ibm', 'text': 'IBM'},
{'id': 'nec', 'text': 'NEC'}], companies)
response = self.app.get('/api/1.0/companies?metric=commits&'
- 'company_name=ib')
+ 'company_name=ib&module=glance')
companies = json.loads(response.data)['companies']
self.assertEqual([{'id': 'ibm', 'text': 'IBM'}], companies)
def test_get_company(self):
with test_api.make_runtime_storage(
- {'repos': [{'module': 'nova', 'project_type': 'openstack',
- 'organization': 'openstack',
+ {'repos': [{'module': 'nova', 'organization': 'openstack',
'uri': 'git://github.com/openstack/nova.git'},
- {'module': 'glance', 'project_type': 'openstack',
- 'organization': 'openstack',
+ {'module': 'glance', 'organization': 'openstack',
'uri': 'git://github.com/openstack/glance.git'}]},
test_api.make_records(record_type=['commit'],
loc=[10, 20, 30],
module=['glance'],
company_name=['NEC', 'IBM', 'NTT'])):
- response = self.app.get('/api/1.0/companies/nec')
+ response = self.app.get('/api/1.0/companies/nec?module=glance')
company = json.loads(response.data)['company']
self.assertEqual({'id': 'nec', 'text': 'NEC'}, company)
- response = self.app.get('/api/1.0/companies/google')
+ response = self.app.get('/api/1.0/companies/google?module=glance')
self.assertEqual(404, response.status_code)
diff --git a/tests/api/test_modules.py b/tests/api/test_modules.py
index d8948650e..55f6c4977 100644
--- a/tests/api/test_modules.py
+++ b/tests/api/test_modules.py
@@ -22,11 +22,9 @@ class TestAPIModules(test_api.TestAPI):
def test_get_modules(self):
with test_api.make_runtime_storage(
- {'repos': [{'module': 'nova', 'project_type': 'openstack',
- 'organization': 'openstack',
+ {'repos': [{'module': 'nova', 'organization': 'openstack',
'uri': 'git://github.com/openstack/nova.git'},
- {'module': 'glance', 'project_type': 'openstack',
- 'organization': 'openstack',
+ {'module': 'glance', 'organization': 'openstack',
'uri': 'git://github.com/openstack/glance.git'}],
'module_groups': [
{'module_group_name': 'nova-group',
@@ -37,7 +35,9 @@ class TestAPIModules(test_api.TestAPI):
response = self.app.get('/api/1.0/modules')
modules = json.loads(response.data)['modules']
self.assertEqual(
- [{'id': 'glance', 'modules': ['glance'], 'text': 'glance'},
+ [{'group': True, 'id': 'all', 'text': 'All',
+ 'modules': ['glance', 'nova']},
+ {'id': 'glance', 'modules': ['glance'], 'text': 'glance'},
{'id': 'nova', 'modules': ['nova'], 'text': 'nova'},
{'group': True, 'id': 'nova-group', 'text': 'nova-group',
'modules': ['nova', 'python-novaclient']}], modules)
@@ -50,8 +50,7 @@ class TestAPIModules(test_api.TestAPI):
def test_get_module(self):
with test_api.make_runtime_storage(
- {'repos': [{'module': 'nova', 'project_type': 'openstack',
- 'organization': 'openstack',
+ {'repos': [{'module': 'nova', 'organization': 'openstack',
'uri': 'git://github.com/openstack/nova.git'}],
'module_groups': [
{'module_group_name': 'nova-group',
diff --git a/tests/api/test_stats.py b/tests/api/test_stats.py
index 9845d9b1b..d35b6e3bc 100644
--- a/tests/api/test_stats.py
+++ b/tests/api/test_stats.py
@@ -22,12 +22,13 @@ class TestAPIStats(test_api.TestAPI):
def test_get_modules(self):
with test_api.make_runtime_storage(
- {'repos': [{'module': 'nova', 'project_type': 'openstack',
- 'organization': 'openstack',
+ {'repos': [{'module': 'nova', 'organization': 'openstack',
'uri': 'git://github.com/openstack/nova.git'},
- {'module': 'glance', 'project_type': 'openstack',
- 'organization': 'openstack',
- 'uri': 'git://github.com/openstack/glance.git'}]},
+ {'module': 'glance', 'organization': 'openstack',
+ 'uri': 'git://github.com/openstack/glance.git'}],
+ 'module_groups': [
+ {'module_group_name': 'openstack',
+ 'modules': ['nova', 'glance']}]},
test_api.make_records(record_type=['commit'],
loc=[10, 20, 30],
module=['nova']),
@@ -50,6 +51,9 @@ class TestAPIStats(test_api.TestAPI):
{'module': 'glance', 'project_type': 'openstack',
'organization': 'openstack',
'uri': 'git://github.com/openstack/glance.git'}],
+ 'module_groups': [
+ {'module_group_name': 'openstack',
+ 'modules': ['nova', 'glance']}],
'user:john_doe': {
'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe',
'companies': [{'company_name': 'NEC', 'end_date': 0}],
@@ -86,6 +90,9 @@ class TestAPIStats(test_api.TestAPI):
{'module': 'glance', 'project_type': 'openstack',
'organization': 'openstack',
'uri': 'git://github.com/openstack/glance.git'}],
+ 'module_groups': [
+ {'module_group_name': 'openstack',
+ 'modules': ['nova', 'glance']}],
'user:john_doe': {
'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe',
'companies': [{'company_name': 'NEC', 'end_date': 0}],
diff --git a/tests/api/test_users.py b/tests/api/test_users.py
index 11ab530fe..2c4cef58c 100644
--- a/tests/api/test_users.py
+++ b/tests/api/test_users.py
@@ -27,12 +27,11 @@ class TestAPIUsers(test_api.TestAPI):
def test_users(self):
with test_api.make_runtime_storage(
- {'repos': [{'module': 'nova', 'project_type': 'openstack',
- 'organization': 'openstack',
+ {'repos': [{'module': 'nova', 'organization': 'openstack',
'uri': 'git://github.com/openstack/nova.git'}]},
- test_api.make_records(record_type=['commit'],
+ test_api.make_records(record_type=['commit'], module=['nova'],
user_id=['john_doe', 'bill_smith'])):
- response = self.app.get('/api/1.0/users')
+ response = self.app.get('/api/1.0/users?module=nova')
users = json.loads(response.data)['users']
self.assertEqual(2, len(users))
self.assertIn({'id': 'john_doe', 'text': 'John Doe'}, users)
@@ -40,12 +39,11 @@ class TestAPIUsers(test_api.TestAPI):
def test_users_search(self):
with test_api.make_runtime_storage(
- {'repos': [{'module': 'nova', 'project_type': 'openstack',
- 'organization': 'openstack',
+ {'repos': [{'module': 'nova', 'organization': 'openstack',
'uri': 'git://github.com/openstack/nova.git'}]},
- test_api.make_records(record_type=['commit'],
+ test_api.make_records(record_type=['commit'], module=['nova'],
user_name=['John Doe', 'Bill Smith'])):
- response = self.app.get('/api/1.0/users?query=doe')
+ response = self.app.get('/api/1.0/users?module=nova&query=doe')
users = json.loads(response.data)['users']
self.assertEqual(1, len(users))
self.assertIn({'id': 'john_doe', 'text': 'John Doe'}, users)
@@ -55,7 +53,9 @@ class TestAPIUsers(test_api.TestAPI):
{'user:john_doe': {
'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe',
'companies': [{'company_name': 'NEC', 'end_date': 0}],
- 'emails': 'john_doe@gmail.com'}}):
+ 'emails': 'john_doe@gmail.com'}},
+ test_api.make_records(record_type=['commit'], module=['nova'],
+ user_name=['John Doe', 'Bill Smith'])):
response = self.app.get('/api/1.0/users/john_doe')
user = json.loads(response.data)['user']
self.assertEqual('john_doe', user['user_id'])
@@ -65,6 +65,12 @@ class TestAPIUsers(test_api.TestAPI):
{'user:john_doe': {
'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe',
'companies': [{'company_name': 'NEC', 'end_date': 0}],
- 'emails': 'john_doe@gmail.com'}}):
+ 'emails': 'john_doe@gmail.com'},
+ 'repos': [{'module': 'nova', 'organization': 'openstack',
+ 'uri': 'git://github.com/openstack/nova.git'}],
+ 'module_groups': [
+ {'module_group_name': 'openstack', 'modules': ['nova']}]},
+ test_api.make_records(record_type=['commit'], module=['nova'],
+ user_name=['John Doe', 'Bill Smith'])):
response = self.app.get('/api/1.0/users/nonexistent')
self.assertEqual(404, response.status_code)
diff --git a/tests/unit/test_default_data_processor.py b/tests/unit/test_default_data_processor.py
index e7eea4a11..9e72e3a9f 100644
--- a/tests/unit/test_default_data_processor.py
+++ b/tests/unit/test_default_data_processor.py
@@ -56,18 +56,32 @@ class TestDefaultDataProcessor(testtools.TestCase):
with mock.patch('stackalytics.processor.default_data_processor.'
'_retrieve_project_list_from_github') as retriever:
retriever.return_value = [
- {'module': 'nova', 'uri': 'git://github.com/openstack/nova'},
- {'module': 'qa', 'uri': 'git://github.com/openstack/qa'},
+ {'module': 'nova', 'uri': 'git://github.com/openstack/nova',
+ 'organization': 'openstack'},
+ {'module': 'qa', 'uri': 'git://github.com/openstack/qa',
+ 'organization': 'openstack'},
]
dd = {
'repos': [
- {'module': 'qa', 'uri': 'git://github.com/openstack/qa'},
+ {'module': 'qa', 'uri': 'git://github.com/openstack/qa',
+ 'organization': 'openstack'},
+ {'module': 'tux', 'uri': 'git://github.com/stackforge/tux',
+ 'organization': 'stackforge'},
],
- 'project_sources': ['any']
+ 'project_sources': [{'organization': 'openstack',
+ 'module_group_name': 'OpenStack'}],
+ 'module_groups': [],
}
default_data_processor._update_project_list(dd)
- self.assertEqual(2, len(dd['repos']))
+ self.assertEqual(3, len(dd['repos']))
self.assertIn('qa', set([r['module'] for r in dd['repos']]))
self.assertIn('nova', set([r['module'] for r in dd['repos']]))
+ self.assertIn('tux', set([r['module'] for r in dd['repos']]))
+
+ self.assertEqual(2, len(dd['module_groups']))
+ self.assertIn({'module_group_name': 'OpenStack',
+ 'modules': ['qa', 'nova']}, dd['module_groups'])
+ self.assertIn({'module_group_name': 'stackforge',
+ 'modules': ['tux']}, dd['module_groups'])