diff --git a/etc/default_data.json b/etc/default_data.json index 47f573c38..c7cc7c47f 100644 --- a/etc/default_data.json +++ b/etc/default_data.json @@ -14141,19 +14141,13 @@ { "id": "openstack", "title": "OpenStack", - "modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"] + "modules": ["tc-approved-release", "governance", "api-wg"] }, { - "id": "integrated", - "title": "integrated", + "id": "tc-approved-release", + "title": "tc-approved", "child": true, - "modules": ["official-integrated"] - }, - { - "id": "other", - "title": "other", - "child": true, - "modules": ["official-other"] + "modules": ["tc-approved-release"] }, { "id": "stackforge", diff --git a/etc/test_default_data.json b/etc/test_default_data.json index 17107e10e..b095ecaac 100644 --- a/etc/test_default_data.json +++ b/etc/test_default_data.json @@ -168,6 +168,10 @@ { "release_name": "Kilo", "end_date": "2015-Apr-30" + }, + { + "release_name": "Liberty", + "end_date": "2015-Oct-15" } ], @@ -183,19 +187,13 @@ { "id": "openstack", "title": "OpenStack", - "modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"] + "modules": ["tc-approved-release", "governance", "python-openstackclient", "api-wg"] }, { - "id": "integrated", - "title": "integrated", + "id": "tc-approved-release", + "title": "tc-approved", "child": true, - "modules": ["official-integrated"] - }, - { - "id": "other", - "title": "other", - "child": true, - "modules": ["official-other"] + "modules": ["tc-approved-release"] }, { "id": "stackforge", diff --git a/etc/test_projects.yaml b/etc/test_projects.yaml index a11b3bc72..48c7df7bb 100644 --- a/etc/test_projects.yaml +++ b/etc/test_projects.yaml @@ -1,16 +1,38 @@ Sahara: ptl: Sergey Lukjanov (SergeyLukjanov) + irc-channel: openstack-sahara + service: Data processing service mission: > To provide a scalable data processing stack and associated management interfaces. url: https://wiki.openstack.org/wiki/Sahara - projects: - - repo: openstack/sahara + deliverables: + python-saharaclient: + repos: + - openstack/python-saharaclient tags: - - name: integrated-release - since: juno - - repo: openstack/python-saharaclient - - repo: openstack/sahara-dashboard - - repo: openstack/sahara-extra - - repo: openstack/sahara-image-elements - - repo: openstack/sahara-specs + - release:cycle-with-intermediary + - release:has-stable-branches + - type:library + - release:managed + - vulnerability:managed + sahara: + repos: + - openstack/sahara + - openstack/sahara-extra + - openstack/sahara-image-elements + tags: + - tc-approved-release + - release:managed + - release:cycle-with-milestones + - release:has-stable-branches + - type:service + - vulnerability:managed + sahara-dashboard: + repos: + - openstack/sahara-dashboard + tags: + - type:library + sahara-specs: + repos: + - openstack/sahara-specs diff --git a/stackalytics/processor/governance.py b/stackalytics/processor/governance.py new file mode 100644 index 000000000..f72ccc563 --- /dev/null +++ b/stackalytics/processor/governance.py @@ -0,0 +1,62 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from oslo_log import log as logging +import six +import yaml + +from stackalytics.processor import utils + + +LOG = logging.getLogger(__name__) + +TAGS = ['tc-approved-release'] # list of supported tags + + +def read_projects_yaml(project_list_uri): + LOG.debug('Process list of projects from uri: %s', project_list_uri) + content = yaml.safe_load(utils.read_uri(project_list_uri)) + module_groups = collections.defaultdict(lambda: {'modules': []}) + + for tag in TAGS: + m = module_groups[tag] # object created by defaultdict + m['tag'] = 'project_type' + m['module_group_name'] = tag + + for name, project in six.iteritems(content): + group_id = '%s-group' % name.lower() + module_groups[group_id]['module_group_name'] = '%s Official' % name + module_groups[group_id]['tag'] = 'program' + + for d_name, deliverable in six.iteritems(project['deliverables']): + for repo in deliverable['repos']: + repo_split = repo.split('/') + if len(repo_split) < 2: + continue # valid repo must be in form of 'org/module' + module_name = repo_split[1] + + module_groups[group_id]['modules'].append(module_name) + + tags = deliverable.get('tags', []) + for tag in tags: + if tag in TAGS: + module_groups[tag]['modules'].append(module_name) + + # set ids for module groups + for group_id, group in six.iteritems(module_groups): + group['id'] = group_id + group['modules'].sort() + + return module_groups diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 24a5b13bd..fa1b87d0c 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -13,19 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections - from oslo_config import cfg from oslo_log import log as logging import psutil -import six from six.moves.urllib import parse -import yaml from stackalytics.processor import bps from stackalytics.processor import config from stackalytics.processor import default_data_processor from stackalytics.processor import driverlog +from stackalytics.processor import governance from stackalytics.processor import lp from stackalytics.processor import mls from stackalytics.processor import mps @@ -35,7 +32,6 @@ from stackalytics.processor import runtime_storage from stackalytics.processor import utils from stackalytics.processor import vcs - LOG = logging.getLogger(__name__) @@ -242,66 +238,10 @@ def apply_corrections(uri, runtime_storage_inst): runtime_storage_inst.apply_corrections(valid_corrections) -def _read_official_projects_yaml(project_list_uri, release_names): - LOG.debug('Process list of projects from uri: %s', project_list_uri) - content = yaml.safe_load(utils.read_uri(project_list_uri)) - module_groups = collections.defaultdict( - lambda: {'modules': [], 'releases': collections.defaultdict(list)}) - - official_integrated = module_groups['official-integrated'] - official_integrated['tag'] = 'project_type' - official_integrated['module_group_name'] = 'official-integrated' - official_other = module_groups['official-other'] - official_other['tag'] = 'project_type' - official_other['module_group_name'] = 'official-other' - - for name, info in six.iteritems(content): - # take one official project - - group_id = '%s-group' % name.lower() - module_groups[group_id]['module_group_name'] = '%s Official' % name - module_groups[group_id]['tag'] = 'program' - - for module in info['projects']: - repo_split = module['repo'].split('/') - if len(repo_split) < 2: - continue # valid repo must be in form of 'org/module' - module_name = repo_split[1] - - module_groups[group_id]['modules'].append(module_name) - - type_matched = False - if 'tags' in module: - for tag in module.get('tags'): - tag_name = tag.get('name') - - if tag_name == 'integrated-release': - type_matched = True # project type is matched here - project_type = 'official-other' - for release_name in release_names: - if release_name == tag.get('since'): - project_type = 'official-integrated' - - module_groups[project_type]['releases'][ - release_name].append(module_name) - - if not type_matched: - module_groups['official-other']['modules'].append(module_name) - - # set ids for module groups - for group_id, group in six.iteritems(module_groups): - group['id'] = group_id - - return module_groups - - def process_project_list(runtime_storage_inst, project_list_uri): module_groups = runtime_storage_inst.get_by_key('module_groups') or {} - release_names = [r['release_name'].lower() - for r in runtime_storage_inst.get_by_key('releases')[1:]] - official_module_groups = _read_official_projects_yaml( - project_list_uri, release_names) + official_module_groups = governance.read_projects_yaml(project_list_uri) LOG.debug('Update module groups with official: %s', official_module_groups) module_groups.update(official_module_groups) diff --git a/stackalytics/processor/rcs.py b/stackalytics/processor/rcs.py index aa3f20fc7..0104c8b12 100644 --- a/stackalytics/processor/rcs.py +++ b/stackalytics/processor/rcs.py @@ -126,6 +126,7 @@ class Gerrit(Rcs): start_id=0, last_id=0, is_open=False, grab_comments=False): sort_key = start_id + last_id = last_id or 0 while True: cmd = self._get_cmd(project_organization, module, branch, sort_key, diff --git a/tests/unit/test_governance.py b/tests/unit/test_governance.py new file mode 100644 index 000000000..34c19e302 --- /dev/null +++ b/tests/unit/test_governance.py @@ -0,0 +1,87 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import testtools + +from stackalytics.processor import governance + + +SAMPLE = """ +Sahara: + ptl: Sergey Lukjanov (SergeyLukjanov) + irc-channel: openstack-sahara + service: Data processing service + mission: > + To provide a scalable data processing stack and associated management + interfaces. + url: https://wiki.openstack.org/wiki/Sahara + deliverables: + python-saharaclient: + repos: + - openstack/python-saharaclient + tags: + - release:cycle-with-intermediary + - release:has-stable-branches + - type:library + - release:managed + - vulnerability:managed + sahara: + repos: + - openstack/sahara + - openstack/sahara-extra + - openstack/sahara-image-elements + tags: + - tc-approved-release + - release:managed + - release:cycle-with-milestones + - release:has-stable-branches + - type:service + - vulnerability:managed + sahara-dashboard: + repos: + - openstack/sahara-dashboard + tags: + - type:library + sahara-specs: + repos: + - openstack/sahara-specs +""" + + +class TestGovernance(testtools.TestCase): + + @mock.patch('stackalytics.processor.utils.read_uri') + def test_read_official_projects_yaml(self, read_uri): + read_uri.return_value = SAMPLE + + expected = { + 'sahara-group': { + 'id': 'sahara-group', + 'module_group_name': 'Sahara Official', + 'modules': ['python-saharaclient', 'sahara', + 'sahara-dashboard', 'sahara-extra', + 'sahara-image-elements', 'sahara-specs'], + 'tag': 'program' + }, + 'tc-approved-release': { + 'id': 'tc-approved-release', + 'module_group_name': 'tc-approved-release', + 'modules': ['sahara', 'sahara-extra', 'sahara-image-elements'], + 'tag': 'project_type' + } + } + + actual = governance.read_projects_yaml('uri') + + self.assertEqual(expected, actual)