Project type is back to UI to keep compatibility with 0.4

* To keep UI compatibility with version 0.4 project type is returned.
List of project types is configured in default data, every type is
a list of modules or module groups.
* Module groups now have tags to distinguish between different kinds.
Introduced:
  'group' - for user-configured groups,
  'program' - for official programs,
  'project_type' - for types of projects within official programs,
  'organization' - for module groups generated for github organization

Change-Id: I8d5e46e18c7327e8c9d114e0a5eec021305b843e

dashboard/decorators.py

@@ -69,6 +69,14 @@ def record_filter(ignore=None, use_default=True):
                         memory_storage_inst.get_record_ids_by_modules(
                             vault.resolve_modules(param)))
 
+            if 'project_type' not in ignore:
+                param = parameters.get_parameter(kwargs, 'project_type',
+                                                 'project_types', use_default)
+                if param:
+                    record_ids &= (
+                        memory_storage_inst.get_record_ids_by_modules(
+                            vault.resolve_project_types(param)))
+
             if 'user_id' not in ignore:
                 param = parameters.get_parameter(kwargs, 'user_id', 'user_ids')
                 param = [u for u in param
@@ -157,22 +165,25 @@ def mark_finalize(record):
     new_record = record.copy()
 
     positive = 0
+    numeric = 0
     mark_distribution = []
     for key in [-2, -1, 1, 2, 'A']:
         if key in record:
             if key in [1, 2]:
                 positive += record[key]
+            if key in [-2, -1, 1, 2]:
+                numeric += record[key]
             mark_distribution.append(str(record[key]))
         else:
             mark_distribution.append('0')
             new_record[key] = 0
 
     new_record['disagreements'] = record.get('disagreements', 0)
-    if record['metric']:
+    if numeric:
         positive_ratio = '%.1f%%' % (
-            (positive * 100.0) / record['metric'])
+            (positive * 100.0) / numeric)
         new_record['disagreement_ratio'] = '%.1f%%' % (
-            (record.get('disagreements', 0) * 100.0) / record['metric'])
+            (record.get('disagreements', 0) * 100.0) / numeric)
     else:
         positive_ratio = helpers.INFINITY_HTML
         new_record['disagreement_ratio'] = helpers.INFINITY_HTML
@@ -251,6 +262,11 @@ def templated(template=None, return_code=200):
             ctx['metric'] = metric or parameters.get_default('metric')
             ctx['metric_label'] = parameters.METRIC_LABELS[ctx['metric']]
 
+            project_type = flask.request.args.get('project_type')
+            if not vault.is_project_type_valid(project_type):
+                project_type = parameters.get_default('project_type')
+            ctx['project_type'] = project_type
+
             release = flask.request.args.get('release')
             releases = vault_inst['releases']
             if release:
@@ -265,6 +281,7 @@ def templated(template=None, return_code=200):
             ctx['review_nth'] = (flask.request.args.get('review_nth') or
                                  parameters.get_default('review_nth'))
 
+            ctx['project_type_options'] = vault.get_project_types()
             ctx['release_options'] = vault.get_release_options()
             ctx['metric_options'] = sorted(parameters.METRIC_LABELS.items(),
                                            key=lambda x: x[0])
@@ -276,7 +293,8 @@ def templated(template=None, return_code=200):
 
             module = parameters.get_single_parameter(kwargs, 'module')
             ctx['module'] = module
-            ctx['module_inst'] = vault_inst['module_id_index'][module]
+            if module:
+                ctx['module_inst'] = vault_inst['module_id_index'][module]
 
             ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id')
             ctx['page_title'] = helpers.make_page_title(

dashboard/helpers.py

@@ -178,7 +178,8 @@ def format_launchpad_module_link(module):
 
 
 def make_link(title, uri=None, options=None):
-    param_names = ('release', 'module', 'company', 'user_id', 'metric')
+    param_names = ('release', 'project_type', 'module', 'company', 'user_id',
+                   'metric')
     param_values = {}
     for param_name in param_names:
         v = parameters.get_parameter({}, param_name, param_name)

dashboard/parameters.py

@@ -25,7 +25,7 @@ LOG = logging.getLogger(__name__)
 DEFAULTS = {
     'metric': 'commits',
     'release': 'icehouse',
-    'module': 'openstack',
+    'project_type': 'openstack',
     'review_nth': 5,
 }
 

dashboard/static/css/style.css

@@ -235,11 +235,16 @@ a[href^="https://launchpad"]:after {
     font-size: 9pt;
 }
 
-.select_group {
+.select_module_group {
     font-weight: bold;
     color: #4bb2c5;
 }
 
+.select_module_program {
+    font-weight: bold;
+    color: #ab64c5;
+}
+
 .project_group {
     font-weight: bold;
 }

dashboard/static/js/stackalytics-ui.js

@@ -276,6 +276,7 @@ function make_std_options() {
     var options = {};
     options['release'] = $('#release').val();
     options['metric'] = $('#metric').val();
+    options['project_type'] = $('#project_type').val();
     options['module'] = $('#module').val() || '';
     options['company'] = $('#company').val() || '';
     options['user_id'] = $('#user').val() || '';
@@ -370,11 +371,11 @@ function init_selectors(base_url) {
             results: function (data, page) {
                 const project_types = data["project_types"];
                 var result = [];
-                result.push({id: "all", text: "all", group: true});
                 for (var key in project_types) {
                     result.push({id: project_types[key].id, text: project_types[key].text, group: true});
                     for (var i in project_types[key].items) {
-                        result.push({id: project_types[key].items[i], text: project_types[key].items[i]});
+                        var item = project_types[key].items[i];
+                        result.push({id: item.id, text: item.text});
                     }
                 }
                 return {results: result};
@@ -435,13 +436,13 @@ function init_selectors(base_url) {
         });
 
     $("#module").select2({
-        allowClear: false,
+        allowClear: true,
         ajax: {
-            url: make_uri(base_url + "/api/1.0/modules"),
+            url: make_uri(base_url + "/api/1.0/modules", {tags: "module,program,group"}),
             dataType: 'jsonp',
             data: function (term, page) {
                 return {
-                    module_name: term
+                    query: term
                 };
             },
             results: function (data, page) {
@@ -459,8 +460,8 @@ function init_selectors(base_url) {
             }
         },
         formatResultCssClass: function (item) {
-            if (item.group) {
+            if (item.tag) {
-                return "select_group"
+                return "select_module_" + item.tag;
             }
             return "";
         }

dashboard/templates/layout.html

@@ -59,7 +59,7 @@
             <a href="https://wiki.openstack.org/wiki/Stackalytics" title="Version {{ stackalytics_version }} ({{ stackalytics_release }})">About ↗</a>
         </div>
         <div id="analytics_header">
-            <span id="logo"><a href="/?metric={{ metric }}&release={{ release }}">Stackalytics</a></span>
+            <span id="logo"><a href="/?metric={{ metric }}&release={{ release }}&project_type={{ project_type }}">Stackalytics</a></span>
             <span id="slogan">| community heartbeat</span>
         </div>
 
@@ -67,27 +67,32 @@
 
             <div class="drop">
                 <label for="release">Release</label>
-                <input type="hidden" id="release" style="width: 170px" data-placeholder="Select release"/>
+                <input type="hidden" id="release" style="width: 140px" data-placeholder="Select release"/>
+            </div>
+
+            <div class="drop">
+                <label for="project_type">Projects</label>
+                <input type="hidden" id="project_type" style="width: 140px" data-placeholder="Select project type"/>
             </div>
 
             <div class="drop">
                 <label for="module">Module</label>
-                <input type="hidden" id="module" style="width: 170px" data-placeholder="Any module" value="{{ module }}"/>
+                <input type="hidden" id="module" style="width: 140px" data-placeholder="Any module" value="{{ module }}"/>
             </div>
 
             <div class="drop">
                 <label for="company">Company</label>
-                <input type="hidden" id="company" style="width: 170px" data-placeholder="Any company" value="{{ company }}"/>
+                <input type="hidden" id="company" style="width: 140px" data-placeholder="Any company" value="{{ company }}"/>
             </div>
 
             <div class="drop">
                 <label for="company">Engineer</label>
-                <input type="hidden" id="user" style="width: 170px" data-placeholder="Any engineer" value="{{ user_id }}"/>
+                <input type="hidden" id="user" style="width: 140px" data-placeholder="Any engineer" value="{{ user_id }}"/>
             </div>
 
             <div class="drop">
                 <label for="metric">Metric</label>
-                <input type="hidden" id="metric" style="width: 170px" data-placeholder="Select metric"/>
+                <input type="hidden" id="metric" style="width: 140px" data-placeholder="Select metric"/>
             </div>
 
         </div>

dashboard/templates/overview.html

@@ -4,9 +4,9 @@
 {% set show_company_breakdown = (not company) and (not user_id) %}
 {% set show_engineer_breakdown = (not user_id) %}
 {% set show_bp_breakdown = (metric in ['bpd', 'bpc']) %}
-{% set show_module_breakdown = (not module) or (module_inst['group']) %}
+{% set show_module_breakdown = (not module) %}
 {% set show_user_activity = (user_id) %}
-{% set show_module_activity = (module) and (not user_id) and (not module_inst['group']) %}
+{% set show_module_activity = (module) and (not user_id) %}
 {% set show_activity = (show_user_activity) or (show_module_activity) %}
 {% set show_user_contribution = (user_id) or (company) %}
 {% set show_module_contribution = (module) and (not user_id) %}

dashboard/templates/reports/contribution.html

@@ -7,12 +7,12 @@ Contribution into {{ module }} for the last {{ days }} days
 {% block scripts %}
     <script type="text/javascript">
         $(document).ready(function () {
-            var table_column_names = ["index", "link", "mark", "-2", "-1", "1", "2", "A", "positive_ratio", "disagreements", "disagreement_ratio",
+            var table_column_names = ["index", "link", "metric", "-2", "-1", "1", "2", "A", "positive_ratio", "disagreements", "disagreement_ratio",
                                       "review_ratio", "commit", "email"];
             var table_id = "review_stats_table";
 
             $.ajax({
-                url: make_uri("/api/1.0/stats/engineers_extended?metric=marks&module={{ module }}&release=all&start_date={{ start_date }}"),
+                url: make_uri("/api/1.0/stats/engineers_extended?project_type=all&metric=marks&module={{ module }}&release=all&start_date={{ start_date }}"),
                 dataType: "json",
                 success: function (data) {
                     var tableData = data["stats"];
@@ -21,7 +21,7 @@ Contribution into {{ module }} for the last {{ days }} days
                     var sort_by_column = 0;
                     for (var i = 0; i < table_column_names.length; i++) {
                         tableColumns.push({"mData": table_column_names[i]});
-                        if (table_column_names[i] == "mark") {
+                        if (table_column_names[i] == "metric") {
                             sort_by_column = i;
                         }
                     }

dashboard/templates/widget.html

@@ -121,8 +121,8 @@
         </div>
 
         <div class="drop" style="margin-right: 15px;">
-            <label for="module">Modules</label>
+            <label for="project_type">Projects</label>
-            <input type="hidden" id="modules" style="width: 95px" data-placeholder="Select modules"/>
+            <input type="hidden" id="project_type" style="width: 95px" data-placeholder="Select project type"/>
         </div>
 
         <div class="drop">

dashboard/vault.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import os
 
 import flask
@@ -55,6 +56,7 @@ def get_vault():
         if have_updates:
             init_releases(vault)
             init_module_groups(vault)
+            init_project_types(vault)
 
     return vault
 
@@ -81,45 +83,67 @@ def init_releases(vault):
     vault['releases'] = releases_map
 
 
+def _make_module(module_id, text, modules, tag):
+    return {'id': module_id, 'text': text,
+            'modules': modules, 'tag': tag}
+
+
 def init_module_groups(vault):
     runtime_storage_inst = vault['runtime_storage']
     memory_storage_inst = vault['memory_storage']
 
-    module_index = {}
+    module_group_index = collections.defaultdict(set)
     module_id_index = {}
-    module_groups = runtime_storage_inst.get_by_key('module_groups') or []
+    module_groups = runtime_storage_inst.get_by_key('module_groups') or {}
 
-    module_groups.append({'module_group_name': 'All',
+    for module_group in module_groups.values():
-                          'modules': set(memory_storage_inst.get_modules())})
-
-    for module_group in module_groups:
         module_group_name = module_group['module_group_name']
-        module_group_id = module_group_name.lower()
+        module_group_id = module_group.get('id') or module_group_name.lower()
 
-        module_id_index[module_group_id] = {
+        module_id_index[module_group_id] = _make_module(
-            'group': True,
+            module_group_id, module_group_name, module_group['modules'],
-            'id': module_group_id,
+            module_group.get('tag') or 'group')
-            'text': module_group_name,
-            'modules': [m.lower() for m in module_group['modules']],
-        }
 
-        modules = module_group['modules']
+        for module in module_group['modules']:
-        for module in modules:
+            module_group_index[module].add(module_group_id)
-            if module in module_index:
-                module_index[module].add(module_group_id)
-            else:
-                module_index[module] = set([module_group_id])
 
     for module in memory_storage_inst.get_modules():
-        module_id_index[module] = {
+        module_id_index[module] = _make_module(module.lower(), module,
-            'id': module.lower(),
+                                               [module.lower()], 'module')
-            'text': module,
-            'modules': [module.lower()],
-        }
 
-    vault['module_group_index'] = module_index
+    module_id_index['all'] = _make_module('all', 'All',
+                                          memory_storage_inst.get_modules(),
+                                          'project_type')
+
+    vault['module_group_index'] = module_group_index
     vault['module_id_index'] = module_id_index
-    vault['module_groups'] = module_groups
+
+
+def init_project_types(vault):
+    runtime_storage_inst = vault['runtime_storage']
+    project_types = runtime_storage_inst.get_by_key('project_types') or {}
+
+    result = []
+    parent = None
+    for pt in project_types:
+        is_child = pt.get('child', False)
+        if parent and is_child:
+            item = {'id': pt['id'], 'text': pt['title']}
+            if 'items' in parent:
+                parent['items'].append(item)
+            else:
+                parent['items'] = [item]
+        else:
+            parent = pt
+            result.append(parent)
+
+    vault['project_types'] = result
+    vault['project_types_index'] = dict([(pt['id'], pt)
+                                         for pt in project_types])
+
+
+def get_project_types():
+    return get_vault()['project_types']
 
 
 def get_release_options():
@@ -130,6 +154,22 @@ def get_release_options():
     return releases
 
 
+def is_project_type_valid(project_type):
+    if not project_type:
+        return False
+    project_type = project_type.lower()
+    if project_type == 'all':
+        return True
+    project_types = get_vault().get('project_types_index', [])
+    return project_type in project_types
+
+
+def get_project_type(project_type_id):
+    if not is_project_type_valid(project_type_id):
+        return None
+    return get_vault()['project_types_index'][project_type_id]
+
+
 def get_user_from_runtime_storage(user_id):
     runtime_storage_inst = get_vault()['runtime_storage']
     return utils.load_user(runtime_storage_inst, user_id)
@@ -142,3 +182,12 @@ def resolve_modules(module_ids):
         if module_id in module_id_index:
             modules |= set(module_id_index[module_id]['modules'])
     return modules
+
+
+def resolve_project_types(project_types):
+    modules = set()
+    for pt in project_types:
+        if is_project_type_valid(pt):
+            modules |= resolve_modules(
+                get_vault()['project_types_index'][pt]['modules'])
+    return modules

dashboard/web.py

@@ -20,6 +20,7 @@ import time
 
 import flask
 from flask.ext import gravatar as gravatar_ext
+import itertools
 from oslo.config import cfg
 import six
 
@@ -174,7 +175,6 @@ def get_engineers_extended(records):
         result_row[record_type] = result_row.get(record_type, 0) + 1
         if record_type == 'mark':
             decorators.mark_filter(result, record, param_id)
-            result_row['metric'] = result_row['mark']
         if record_type == 'review':
             result_row['patch_count'] = (result_row.get('patch_count', 0) +
                                          record['patch_count'])
@@ -255,27 +255,29 @@ def get_modules_json(records):
     module_group_index = vault.get_vault()['module_group_index']
     module_id_index = vault.get_vault()['module_id_index']
 
-    modules_set = set()
+    tags = parameters.get_parameter({}, 'tag', 'tags')
-    for record in records:
-        module = record['module']
-        if module not in modules_set:
-            modules_set.add(module)
 
-    modules_groups_set = set()
+    # all modules mentioned in records
-    for module in modules_set:
+    module_ids = set(record['module'] for record in records)
-        if module in module_group_index:
+    # plus all module groups that hold these modules
-            modules_groups_set |= module_group_index[module]
+    module_ids |= set(itertools.chain.from_iterable(
+        module_group_index.get(module, []) for module in module_ids))
+    # keep only modules with specified tags
+    if tags:
+        module_ids = set(module_id for module_id in module_ids
+                         if module_id_index[module_id].get('tag') in tags)
 
-    modules_set |= modules_groups_set
+    query = (flask.request.args.get('query') or '').lower()
+    matched = []
 
-    query = (flask.request.args.get('module_name') or '').lower()
+    for module_id in module_ids:
-    options = []
+        if module_id.find(query) >= 0:
+            module = dict([(k, v) for k, v in
+                           six.iteritems(module_id_index[module_id])
+                           if k not in ['modules']])
+            matched.append(module)
 
-    for module in modules_set:
+    return sorted(matched, key=operator.itemgetter('text'))
-        if module.find(query) >= 0:
-            options.append(module_id_index[module])
-
-    return sorted(options, key=operator.itemgetter('text'))
 
 
 @app.route('/api/1.0/companies/<company_name>')
@@ -405,22 +407,19 @@ def get_metric_json(metric):
 @decorators.jsonify('project_types')
 @decorators.exception_handler()
 def get_project_types_json():
-    return [{'id': m, 'text': m, 'items': list(t)}
+    return [{'id': pt['id'], 'text': pt['title'], 'items': pt.get('items', [])}
-            for m, t in vault.get_project_type_options().iteritems()]
+            for pt in vault.get_project_types()]
 
 
 @app.route('/api/1.0/project_types/<project_type>')
 @decorators.jsonify('project_type')
 @decorators.exception_handler()
 def get_project_type_json(project_type):
-    if project_type != 'all':
+    if not vault.is_project_type_valid(project_type):
-        for pt, groups in vault.get_project_type_options().iteritems():
+        project_type = parameters.get_default('project_type')
-            if (project_type == pt) or (project_type in groups):
-                break
-        else:
-            project_type = parameters.get_default('project_type')
 
-    return {'id': project_type, 'text': project_type}
+    pt = vault.get_project_type(project_type)
+    return {'id': pt['id'], 'text': pt['title']}
 
 
 def _get_date(kwargs, param_name):

etc/default_data.json

@@ -6104,5 +6104,52 @@
             "end_date": "2014-Apr-17"
         }
     ],
-    "mail_lists": ["http://lists.openstack.org/pipermail/openstack-dev/"]
+    "mail_lists": ["http://lists.openstack.org/pipermail/openstack-dev/"],
+    "project_types": [
+        {
+            "id": "all",
+            "title": "All",
+            "modules": ["openstack", "openstack-infra", "openstack-dev", "stackforge"]
+        },
+        {
+            "id": "openstack",
+            "title": "OpenStack",
+            "modules": ["openstack", "openstack-infra", "openstack-dev"]
+        },
+        {
+            "id": "integrated",
+            "title": "integrated",
+            "child": true,
+            "modules": ["official-integrated"]
+        },
+        {
+            "id": "incubated",
+            "title": "incubated",
+            "child": true,
+            "modules": ["official-incubated"]
+        },
+        {
+            "id": "documentation",
+            "title": "documentation",
+            "child": true,
+            "modules": ["documentation"]
+        },
+        {
+            "id": "infra",
+            "title": "infra",
+            "child": true,
+            "modules": ["openstack-infra"]
+        },
+        {
+            "id": "other",
+            "title": "other",
+            "child": true,
+            "modules": ["official-other"]
+        },
+        {
+            "id": "stackforge",
+            "title": "Stackforge",
+            "modules": ["stackforge"]
+        }
+    ]
 }

etc/default_data.schema.json

@@ -1,7 +1,7 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "type": "object",
-    "required": ["users", "releases", "companies", "repos"],
+    "required": ["users", "releases", "companies", "repos", "project_types"],
     "properties": {
         "users": {
             "type": "array",
@@ -174,6 +174,33 @@
             "items": {
                 "type": "string"
             }
+        },
+        "project_types": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "id": {
+                        "type": "string",
+                        "pattern": "^[\\w-]+$"
+                    },
+                    "child": {
+                        "type": "boolean"
+                    },
+                    "title": {
+                        "type": "string"
+                    },
+                    "modules": {
+                        "type": ["array"],
+                        "items": {
+                            "type": "string",
+                            "pattern": "^[\\w-]+$"
+                        }
+                    }
+                },
+                "required": ["id", "title", "modules"],
+                "additionalProperties": false
+            }
         }
     }
 }

etc/test_default_data.json

@@ -146,6 +146,42 @@
         }
     ],
 
-    "mail_lists": ["http://lists.openstack.org/pipermail/openstack-dev/"]
+    "mail_lists": ["http://lists.openstack.org/pipermail/openstack-dev/"],
+
+    "project_types": [
+        {
+            "id": "all",
+            "title": "All",
+            "modules": ["openstack", "openstack-infra", "openstack-dev", "stackforge"]
+        },
+        {
+            "id": "openstack",
+            "title": "OpenStack",
+            "modules": ["openstack", "openstack-infra", "openstack-dev"]
+        },
+        {
+            "id": "integrated",
+            "title": "integrated",
+            "child": true,
+            "modules": ["official-integrated"]
+        },
+        {
+            "id": "incubated",
+            "title": "incubated",
+            "child": true,
+            "modules": ["official-incubated"]
+        },
+        {
+            "id": "other",
+            "title": "other",
+            "child": true,
+            "modules": ["official-other"]
+        },
+        {
+            "id": "stackforge",
+            "title": "Stackforge",
+            "modules": ["stackforge"]
+        }
+    ]
 
 }

stackalytics/processor/default_data_processor.py

@@ -89,13 +89,13 @@ def _create_module_groups(project_sources, repos):
                              for ps in project_sources])
 
     module_groups = []
-    for ogn, modules in organizations.iteritems():
+    for ogn, modules in six.iteritems(organizations):
         if ogn in ps_organizations:
             module_group_name = ps_organizations[ogn]
         else:
             module_group_name = ogn
         module_groups.append({'module_group_name': module_group_name,
-                              'modules': modules})
+                              'modules': modules, 'tag': 'organization'})
 
     return module_groups
 
@@ -130,9 +130,15 @@ def _store_companies(runtime_storage_inst, companies):
     runtime_storage_inst.set_by_key('companies', domains_index)
 
 
+def _store_module_groups(runtime_storage_inst, module_groups):
+    mg_set = dict([(mg['module_group_name'], mg) for mg in module_groups])
+    runtime_storage_inst.set_by_key('module_groups', mg_set)
+
+
 STORE_FUNCS = {
     'users': _store_users,
     'companies': _store_companies,
+    'module_groups': _store_module_groups,
 }
 
 

stackalytics/processor/main.py

@@ -172,8 +172,9 @@ def apply_corrections(uri, runtime_storage_inst):
     runtime_storage_inst.apply_corrections(valid_corrections)
 
 
-def _make_module_group(name, modules):
+def _make_module_group(group_id, name, modules, tag=None):
-    module_group = {'module_group_name': name, 'modules': modules}
+    module_group = {'id': group_id, 'module_group_name': name,
+                    'modules': modules, 'tag': tag}
     LOG.debug('New module group: %s', module_group)
     return module_group
 
@@ -184,8 +185,10 @@ def _read_module_groups(program_list_uri):
     module_groups = []
     modules_by_types = collections.defaultdict(list)
     for name, info in six.iteritems(content):
+        group_id = name.lower()
         if 'codename' in info:
-            name += ' (%s)' % info['codename']
+            name = '%s (%s)' % (info['codename'], name)
+            group_id = '%s-group' % info['codename'].lower()
 
         all_modules = []
         for project_type, project_list in six.iteritems(info['projects']):
@@ -193,30 +196,26 @@ def _read_module_groups(program_list_uri):
             modules_by_types[project_type] += module_list
             all_modules += module_list
 
-        module_groups.append(_make_module_group(name, all_modules))
+        module_groups.append(_make_module_group(
+            group_id, name, all_modules, 'program'))
 
     all_modules = []
     for project_type, modules_list in six.iteritems(modules_by_types):
         all_modules += modules_list
         module_groups.append(
-            _make_module_group('OpenStack ' + project_type.capitalize(),
+            _make_module_group(
-                               modules_list))
+                'official-%s' % project_type, project_type.capitalize(),
-    module_groups.append(_make_module_group('OpenStack All Official',
+                modules_list, 'project_type'))
-                                            all_modules))
+    module_groups.append(_make_module_group(
+        'official-all', 'OpenStack', all_modules, 'project_type'))
     return module_groups
 
 
 def process_program_list(runtime_storage_inst, program_list_uri):
-    stored_module_groups = runtime_storage_inst.get_by_key('module_groups')
+    module_groups = runtime_storage_inst.get_by_key('module_groups') or {}
-    mg_dict = dict([(mg['module_group_name'], mg['modules'])
-                   for mg in stored_module_groups])
     for mg in _read_module_groups(program_list_uri):
-        mg_dict[mg['module_group_name']] = mg['modules']
+        module_groups[mg['module_group_name']] = mg
-
+    runtime_storage_inst.set_by_key('module_groups', module_groups)
-    stored_module_groups = [{'module_group_name': name, 'modules': modules}
-                            for name, modules in six.iteritems(mg_dict)]
-
-    runtime_storage_inst.set_by_key('module_groups', stored_module_groups)
 
 
 def main():

tests/api/test_companies.py

@@ -22,10 +22,20 @@ class TestAPICompanies(test_api.TestAPI):
 
     def test_get_companies(self):
         with test_api.make_runtime_storage(
-                {'repos': [{'module': 'nova', 'organization': 'openstack',
+                {'repos': [{'module': 'nova', 'project_type': 'openstack',
+                            'organization': 'openstack',
                             'uri': 'git://github.com/openstack/nova.git'},
-                           {'module': 'glance', 'organization': 'openstack',
+                           {'module': 'glance', 'project_type': 'openstack',
-                            'uri': 'git://github.com/openstack/glance.git'}]},
+                            'organization': 'openstack',
+                            'uri': 'git://github.com/openstack/glance.git'}],
+                    'project_types': [
+                        {'id': 'openstack', 'title': 'OpenStack',
+                         'modules': ['nova', 'glance']}
+                    ],
+                    'module_groups': {
+                        'openstack': {'module_group_name': 'openstack',
+                                      'modules': ['nova', 'glance']}
+                    }},
                 test_api.make_records(record_type=['commit'],
                                       loc=[10, 20, 30],
                                       module=['glance'],
@@ -63,9 +73,11 @@ class TestAPICompanies(test_api.TestAPI):
 
     def test_get_company(self):
         with test_api.make_runtime_storage(
-                {'repos': [{'module': 'nova', 'organization': 'openstack',
+                {'repos': [{'module': 'nova', 'project_type': 'openstack',
+                            'organization': 'openstack',
                             'uri': 'git://github.com/openstack/nova.git'},
-                           {'module': 'glance', 'organization': 'openstack',
+                           {'module': 'glance', 'project_type': 'openstack',
+                            'organization': 'openstack',
                             'uri': 'git://github.com/openstack/glance.git'}]},
                 test_api.make_records(record_type=['commit'],
                                       loc=[10, 20, 30],

tests/api/test_modules.py

@@ -26,44 +26,47 @@ class TestAPIModules(test_api.TestAPI):
                             'uri': 'git://github.com/openstack/nova.git'},
                            {'module': 'glance', 'organization': 'openstack',
                             'uri': 'git://github.com/openstack/glance.git'}],
-                 'module_groups': [
+                 'module_groups': {'nova-group': {
-                     {'module_group_name': 'nova-group',
+                     'module_group_name': 'nova-group',
-                      'modules': ['nova', 'python-novaclient']}]},
+                     'modules': ['nova', 'python-novaclient']}},
+                 'project_types': [
+                     {'id': 'all', 'title': 'All',
+                      'modules': ['nova', 'glance']}]},
                 test_api.make_records(record_type=['commit'],
                                       module=['glance', 'nova'])):
 
-            response = self.app.get('/api/1.0/modules')
+            response = self.app.get('/api/1.0/modules?project_type=all')
             modules = json.loads(response.data)['modules']
             self.assertEqual(
-                [{'group': True, 'id': 'all', 'text': 'All',
+                [{'id': 'glance', 'text': 'glance', 'tag': 'module'},
-                  'modules': ['glance', 'nova']},
+                 {'id': 'nova', 'text': 'nova', 'tag': 'module'},
-                 {'id': 'glance', 'modules': ['glance'], 'text': 'glance'},
+                 {'id': 'nova-group', 'text': 'nova-group', 'tag': 'group'}],
-                 {'id': 'nova', 'modules': ['nova'], 'text': 'nova'},
+                modules)
-                 {'group': True, 'id': 'nova-group', 'text': 'nova-group',
-                  'modules': ['nova', 'python-novaclient']}], modules)
 
-            response = self.app.get('/api/1.0/modules?module_name=glance')
+            response = self.app.get('/api/1.0/modules?query=glance&'
+                                    'project_type=all')
             modules = json.loads(response.data)['modules']
             self.assertEqual(
-                [{'id': 'glance', 'modules': ['glance'], 'text': 'glance'}],
+                [{'id': 'glance', 'text': 'glance', 'tag': 'module'}],
                 modules)
 
     def test_get_module(self):
         with test_api.make_runtime_storage(
                 {'repos': [{'module': 'nova', 'organization': 'openstack',
                             'uri': 'git://github.com/openstack/nova.git'}],
-                 'module_groups': [
+                 'module_groups': {'nova-group': {
-                     {'module_group_name': 'nova-group',
+                     'module_group_name': 'nova-group',
-                      'modules': ['nova', 'python-novaclient']}]},
+                     'modules': ['nova', 'python-novaclient']}}},
                 test_api.make_records(record_type=['commit'])):
 
             response = self.app.get('/api/1.0/modules/nova')
             module = json.loads(response.data)['module']
             self.assertEqual(
-                {'id': 'nova', 'modules': ['nova'], 'text': 'nova'}, module)
+                {'id': 'nova', 'modules': ['nova'], 'text': 'nova',
+                 'tag': 'module'}, module)
 
             response = self.app.get('/api/1.0/modules/nova-group')
             module = json.loads(response.data)['module']
             self.assertEqual(
-                {'group': True, 'id': 'nova-group', 'text': 'nova-group',
+                {'tag': 'group', 'id': 'nova-group', 'text': 'nova-group',
                  'modules': ['nova', 'python-novaclient']}, module)

tests/api/test_stats.py

@@ -26,8 +26,11 @@ class TestAPIStats(test_api.TestAPI):
                             'uri': 'git://github.com/openstack/nova.git'},
                            {'module': 'glance', 'organization': 'openstack',
                             'uri': 'git://github.com/openstack/glance.git'}],
-                 'module_groups': [
+                 'module_groups': {'openstack': {
-                     {'module_group_name': 'openstack',
+                     'module_group_name': 'openstack',
+                     'modules': ['nova', 'glance']}},
+                 'project_types': [
+                     {'id': 'all', 'title': 'All',
                       'modules': ['nova', 'glance']}]},
                 test_api.make_records(record_type=['commit'],
                                       loc=[10, 20, 30],
@@ -35,7 +38,8 @@ class TestAPIStats(test_api.TestAPI):
                 test_api.make_records(record_type=['commit'],
                                       loc=[100, 200, 300],
                                       module=['glance'])):
-            response = self.app.get('/api/1.0/stats/modules?metric=loc')
+            response = self.app.get('/api/1.0/stats/modules?metric=loc&'
+                                    'project_type=all')
             stats = json.loads(response.data)['stats']
             self.assertEqual(2, len(stats))
             self.assertEqual(600, stats[0]['metric'])
@@ -51,8 +55,11 @@ class TestAPIStats(test_api.TestAPI):
                            {'module': 'glance', 'project_type': 'openstack',
                             'organization': 'openstack',
                             'uri': 'git://github.com/openstack/glance.git'}],
-                 'module_groups': [
+                 'module_groups': {'openstack': {
-                     {'module_group_name': 'openstack',
+                     'module_group_name': 'openstack',
+                     'modules': ['nova', 'glance']}},
+                 'project_types': [
+                     {'id': 'all', 'title': 'All',
                       'modules': ['nova', 'glance']}],
                  'user:john_doe': {
                      'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe',
@@ -77,7 +84,8 @@ class TestAPIStats(test_api.TestAPI):
                                       review_id=['0123456789'],
                                       module=['glance'],
                                       user_id=['john_doe', 'bill'])):
-            response = self.app.get('/api/1.0/stats/engineers?metric=loc')
+            response = self.app.get('/api/1.0/stats/engineers?metric=loc&'
+                                    'project_type=all')
             stats = json.loads(response.data)['stats']
             self.assertEqual(1, len(stats))
             self.assertEqual(660, stats[0]['metric'])
@@ -90,8 +98,11 @@ class TestAPIStats(test_api.TestAPI):
                            {'module': 'glance', 'project_type': 'openstack',
                             'organization': 'openstack',
                             'uri': 'git://github.com/openstack/glance.git'}],
-                 'module_groups': [
+                 'module_groups': {'openstack': {
-                     {'module_group_name': 'openstack',
+                     'module_group_name': 'openstack',
+                     'modules': ['nova', 'glance']}},
+                 'project_types': [
+                     {'id': 'all', 'title': 'All',
                       'modules': ['nova', 'glance']}],
                  'user:john_doe': {
                      'seq': 1, 'user_id': 'john_doe', 'user_name': 'John Doe',
@@ -120,7 +131,8 @@ class TestAPIStats(test_api.TestAPI):
                                       module=['glance'],
                                       author_name=['Bill Smith'],
                                       user_id=['smith'])):
-            response = self.app.get('/api/1.0/stats/engineers_extended')
+            response = self.app.get('/api/1.0/stats/engineers_extended?'
+                                    'project_type=all')
             stats = json.loads(response.data)['stats']
             self.assertEqual(2, len(stats))
             self.assertEqual(2, stats[0]['mark'])

tests/api/test_users.py

@@ -28,7 +28,10 @@ class TestAPIUsers(test_api.TestAPI):
     def test_users(self):
         with test_api.make_runtime_storage(
                 {'repos': [{'module': 'nova', 'organization': 'openstack',
-                            'uri': 'git://github.com/openstack/nova.git'}]},
+                            'uri': 'git://github.com/openstack/nova.git'}],
+                 'project_types': [
+                     {'id': 'openstack', 'title': 'openstack',
+                      'modules': ['nova', 'glance']}]},
                 test_api.make_records(record_type=['commit'], module=['nova'],
                                       user_id=['john_doe', 'bill_smith'])):
             response = self.app.get('/api/1.0/users?module=nova')
@@ -40,7 +43,10 @@ class TestAPIUsers(test_api.TestAPI):
     def test_users_search(self):
         with test_api.make_runtime_storage(
                 {'repos': [{'module': 'nova', 'organization': 'openstack',
-                            'uri': 'git://github.com/openstack/nova.git'}]},
+                            'uri': 'git://github.com/openstack/nova.git'}],
+                 'project_types': [
+                     {'id': 'openstack', 'title': 'openstack',
+                      'modules': ['nova', 'glance']}]},
                 test_api.make_records(record_type=['commit'], module=['nova'],
                                       user_name=['John Doe', 'Bill Smith'])):
             response = self.app.get('/api/1.0/users?module=nova&query=doe')
@@ -68,8 +74,9 @@ class TestAPIUsers(test_api.TestAPI):
                     'emails': 'john_doe@gmail.com'},
                  'repos': [{'module': 'nova', 'organization': 'openstack',
                             'uri': 'git://github.com/openstack/nova.git'}],
-                 'module_groups': [
+                 'module_groups': {'openstack': {
-                     {'module_group_name': 'openstack', 'modules': ['nova']}]},
+                     'module_group_name': 'openstack',
+                     'modules': ['nova']}}},
                 test_api.make_records(record_type=['commit'], module=['nova'],
                                       user_name=['John Doe', 'Bill Smith'])):
             response = self.app.get('/api/1.0/users/nonexistent')

tests/unit/test_default_data_processor.py

@@ -82,6 +82,8 @@ class TestDefaultDataProcessor(testtools.TestCase):
 
             self.assertEqual(2, len(dd['module_groups']))
             self.assertIn({'module_group_name': 'OpenStack',
-                           'modules': ['qa', 'nova']}, dd['module_groups'])
+                           'modules': ['qa', 'nova'],
+                           'tag': 'organization'}, dd['module_groups'])
             self.assertIn({'module_group_name': 'stackforge',
-                           'modules': ['tux']}, dd['module_groups'])
+                           'modules': ['tux'],
+                           'tag': 'organization'}, dd['module_groups'])