diff --git a/etc/default_data.json b/etc/default_data.json index 0533b8fae..30f70a501 100644 --- a/etc/default_data.json +++ b/etc/default_data.json @@ -15460,51 +15460,123 @@ }, { "release_name": "Austin", - "end_date": "2010-Oct-21" + "end_date": "2010-Oct-21", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Bexar", - "end_date": "2011-Feb-03" + "end_date": "2011-Feb-03", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Cactus", - "end_date": "2011-Apr-15" + "end_date": "2011-Apr-15", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Diablo", - "end_date": "2011-Sep-22" + "end_date": "2011-Sep-22", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Essex", - "end_date": "2012-Apr-05" + "end_date": "2012-Apr-05", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Folsom", - "end_date": "2012-Oct-04" + "end_date": "2012-Oct-04", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Grizzly", - "end_date": "2013-Apr-04" + "end_date": "2013-Apr-04", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Havana", - "end_date": "2013-Oct-17" + "end_date": "2013-Oct-17", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Icehouse", - "end_date": "2014-Apr-17" + "end_date": "2014-Apr-17", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Juno", - "end_date": "2014-Oct-16" + "end_date": "2014-Oct-16", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Kilo", - "end_date": "2015-Apr-30" + "end_date": "2015-Apr-30", + "refs": { + "governance": { + "type": "early_big_tent", + "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml?id=c13377dd51e78977d673b64a9a47915b7e4bfe1d" + } + } }, { "release_name": "Liberty", - "end_date": "2015-Oct-15" + "end_date": "2015-Oct-15", + "refs": { + "governance": { + "type": "big_tent", + "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml" + } + } } ], "mail_lists": [ diff --git a/etc/default_data.schema.json b/etc/default_data.schema.json index 844fcb76e..6807970ec 100644 --- a/etc/default_data.schema.json +++ b/etc/default_data.schema.json @@ -64,6 +64,9 @@ }, "end_date": { "$ref": "#/definitions/date_format" + }, + "refs": { + "type": "object" } }, "required": ["release_name", "end_date"], diff --git a/etc/test_default_data.json b/etc/test_default_data.json index 2e2857c5e..18198e4a0 100644 --- a/etc/test_default_data.json +++ b/etc/test_default_data.json @@ -127,51 +127,123 @@ }, { "release_name": "Austin", - "end_date": "2010-Oct-21" + "end_date": "2010-Oct-21", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Bexar", - "end_date": "2011-Feb-03" + "end_date": "2011-Feb-03", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Cactus", - "end_date": "2011-Apr-15" + "end_date": "2011-Apr-15", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Diablo", - "end_date": "2011-Sep-22" + "end_date": "2011-Sep-22", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Essex", - "end_date": "2012-Apr-05" + "end_date": "2012-Apr-05", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Folsom", - "end_date": "2012-Oct-04" + "end_date": "2012-Oct-04", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Grizzly", - "end_date": "2013-Apr-04" + "end_date": "2013-Apr-04", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Havana", - "end_date": "2013-Oct-17" + "end_date": "2013-Oct-17", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Icehouse", - "end_date": "2014-Apr-17" + "end_date": "2014-Apr-17", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Juno", - "end_date": "2014-Oct-16" + "end_date": "2014-Oct-16", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Kilo", - "end_date": "2015-Apr-30" + "end_date": "2015-Apr-30", + "refs": { + "governance": { + "type": "early_big_tent", + "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml?id=c13377dd51e78977d673b64a9a47915b7e4bfe1d" + } + } }, { "release_name": "Liberty", - "end_date": "2015-Oct-15" + "end_date": "2015-Oct-15", + "refs": { + "governance": { + "type": "big_tent", + "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml" + } + } } ], diff --git a/stackalytics/dashboard/templates/layout.html b/stackalytics/dashboard/templates/layout.html index f3dd3bf26..c258140d2 100644 --- a/stackalytics/dashboard/templates/layout.html +++ b/stackalytics/dashboard/templates/layout.html @@ -57,7 +57,7 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
- +
diff --git a/stackalytics/processor/governance.py b/stackalytics/processor/governance.py index b3f700229..3c35c4d51 100644 --- a/stackalytics/processor/governance.py +++ b/stackalytics/processor/governance.py @@ -33,19 +33,51 @@ def _make_module_group(module_groups, name): return m -def read_projects_yaml(project_list_uri): - LOG.debug('Process list of projects from uri: %s', project_list_uri) - content = yaml.safe_load(utils.read_uri(project_list_uri)) - module_groups = collections.defaultdict(lambda: {'modules': []}) +def read_legacy_programs_yaml(module_groups, release_name, content): + all_official = module_groups['openstack-official'] - all_official = _make_module_group(module_groups, 'openstack-official') + for name, info in six.iteritems(content): + # for one program + # group_id = name.lower() + # if 'codename' in info: + # name = '%s (%s)' % (info['codename'], name) + # group_id = '%s-group' % info['codename'].lower() + # + # module_groups[group_id]['module_group_name'] = name + # module_groups[group_id]['tag'] = 'program' - for tag in TAGS: - _make_module_group(module_groups, tag) + for module in info['projects']: + mn = module['repo'].split('/')[1] # module_name + + # module_groups[group_id]['releases'][release_name].append(mn) + all_official['releases'][release_name].append(mn) + + +def read_early_big_tent_projects_yaml(module_groups, release_name, content): + all_official = module_groups['openstack-official'] + + for name, info in six.iteritems(content): + # group_id = '%s-group' % name.lower() + # module_groups[group_id]['module_group_name'] = '%s Official' % name + # module_groups[group_id]['tag'] = 'program' + + for module in info['projects']: + repo_split = module['repo'].split('/') + if len(repo_split) < 2: + continue # valid repo must be in form of 'org/module' + mn = repo_split[1] + + # module_groups[group_id]['releases'][release_name].append(mn) + all_official['releases'][release_name].append(mn) + + +def read_big_tent_projects_yaml(module_groups, release_name, content): + all_official = module_groups['openstack-official'] for name, project in six.iteritems(content): group_id = '%s-group' % name.lower() - module_groups[group_id]['module_group_name'] = '%s Official' % name + module_groups[group_id]['module_group_name'] = ( + '%s Official' % name.title()) module_groups[group_id]['tag'] = 'program' for d_name, deliverable in six.iteritems(project['deliverables']): @@ -53,20 +85,66 @@ def read_projects_yaml(project_list_uri): repo_split = repo.split('/') if len(repo_split) < 2: continue # valid repo must be in form of 'org/module' - module_name = repo_split[1] - module_groups[group_id]['modules'].append(module_name) + mn = repo_split[1] # module_name - all_official['modules'].append(module_name) + module_groups[group_id]['modules'].append(mn) + all_official['releases'][release_name].append(mn) tags = deliverable.get('tags', []) for tag in tags: if tag in TAGS: - module_groups[tag]['modules'].append(module_name) + module_groups[tag]['modules'].append(mn) + + +def _make_default_module_groups(): + # create default module groups + module_groups = collections.defaultdict(lambda: {'modules': []}) + + # openstack official + _make_module_group(module_groups, 'openstack-official') + module_groups['openstack-official']['releases'] = ( + collections.defaultdict(list)) + + # tags + for tag in TAGS: + _make_module_group(module_groups, tag) + + return module_groups + + +GOVERNANCE_PROCESSORS = { + 'legacy': read_legacy_programs_yaml, + 'early_big_tent': read_early_big_tent_projects_yaml, + 'big_tent': read_big_tent_projects_yaml, +} + + +def process_official_list(releases): + module_groups = _make_default_module_groups() + releases_with_refs = (r for r in releases if r.get('refs')) + + for release in releases_with_refs: + ref_governance = release['refs'].get('governance') + if not ref_governance: + continue + + gov_type = ref_governance['type'] + gov_source = ref_governance['source'] + release_name = release['release_name'].lower() + + LOG.debug('Process governance content from uri: %s', gov_source) + content = yaml.safe_load(utils.read_uri(gov_source)) + + GOVERNANCE_PROCESSORS[gov_type](module_groups, release_name, content) # set ids for module groups for group_id, group in six.iteritems(module_groups): group['id'] = group_id group['modules'].sort() + if 'releases' in group: + for gr in six.itervalues(group['releases']): + gr.sort() + return module_groups diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index fa1b87d0c..987e0986b 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -240,8 +240,10 @@ def apply_corrections(uri, runtime_storage_inst): def process_project_list(runtime_storage_inst, project_list_uri): module_groups = runtime_storage_inst.get_by_key('module_groups') or {} + releases = runtime_storage_inst.get_by_key('releases') or {} + + official_module_groups = governance.process_official_list(releases) - official_module_groups = governance.read_projects_yaml(project_list_uri) LOG.debug('Update module groups with official: %s', official_module_groups) module_groups.update(official_module_groups) diff --git a/stackalytics/tests/unit/test_governance.py b/stackalytics/tests/unit/test_governance.py index ca77a509c..5182e6523 100644 --- a/stackalytics/tests/unit/test_governance.py +++ b/stackalytics/tests/unit/test_governance.py @@ -62,7 +62,7 @@ Sahara: class TestGovernance(testtools.TestCase): @mock.patch('stackalytics.processor.utils.read_uri') - def test_read_official_projects_yaml(self, read_uri): + def test_process_official_list(self, read_uri): read_uri.return_value = SAMPLE expected = { @@ -95,13 +95,21 @@ class TestGovernance(testtools.TestCase): 'openstack-official': { 'id': 'openstack-official', 'module_group_name': 'openstack-official', - 'modules': ['python-saharaclient', 'sahara', - 'sahara-dashboard', 'sahara-extra', - 'sahara-image-elements', 'sahara-specs'], + 'modules': [], + 'releases': { + 'liberty': ['python-saharaclient', 'sahara', + 'sahara-dashboard', 'sahara-extra', + 'sahara-image-elements', 'sahara-specs'], + }, 'tag': 'project_type' }, } - actual = governance.read_projects_yaml('uri') + releases = [{ + 'release_name': 'Liberty', + 'refs': {'governance': {'type': 'big_tent', 'source': 'uri'}} + }] + + actual = governance.process_official_list(releases) self.assertEqual(expected, actual)