Merge "Make project list retrieval configurable"

etc/default_data.json

@@ -10301,6 +10301,10 @@
         },
         {
             "organization": "stackforge"
+        },
+        {
+            "organization": "cloudfoundry",
+            "uri": "github://"
         }
     ],
     "module_groups": [
@@ -10440,7 +10444,25 @@
         {
             "id": "complimentary",
             "title": "Complimentary",
-            "modules": ["docker", "kubernetes", "jcloud"]
+            "modules": ["docker", "kubernetes", "jclouds", "cloudfoundry"]
+        },
+        {
+            "id": "cloudfoundry-group",
+            "title": "CloudFoundary",
+            "child": true,
+            "modules": ["cloudfoundry"]
+        },
+        {
+            "id": "docker-group",
+            "title": "Docker",
+            "child": true,
+            "modules": ["docker"]
+        },
+        {
+            "id": "kubernetes-group",
+            "title": "kubernetes",
+            "child": true,
+            "modules": ["kubernetes"]
         }
     ]
 }

etc/default_data.schema.json

@@ -146,9 +146,17 @@
                     "organization": {
                         "type": "string"
                     },
-                    "module_group_name": {
+                    "uri": {
-                        "type": "string",
+                        "type": "string"
-                        "pattern": "^[\\w-]+$"
+                    },
+                    "git_base_uri": {
+                        "type": "string"
+                    },
+                    "ssh_key_filename": {
+                        "type": "string"
+                    },
+                    "ssh_username": {
+                        "type": "string"
                     },
                     "exclude": {
                         "type": "array",

etc/test_default_data.json

@@ -109,7 +109,7 @@
     "project_sources": [
         {
             "organization": "openstack-ops",
-            "module_group_name": "OpenStack-Ops"
+            "uri": "gerrit://review.openstack.org"
         }
     ],
 

requirements-py3.txt

@@ -12,6 +12,7 @@ oslo.serialization>=1.0.0   # Apache-2.0
 oslo.utils>=1.0.0           # Apache-2.0
 paramiko>=1.13.0
 psutil>=1.1.1,<2.0.0
+PyGithub
 python3-memcached>=1.48
 PyYAML>=3.1.0
 sh

requirements.txt

@@ -12,6 +12,7 @@ oslo.serialization>=1.0.0   # Apache-2.0
 oslo.utils>=1.0.0           # Apache-2.0
 paramiko>=1.13.0
 psutil>=1.1.1,<2.0.0
+PyGithub
 python-memcached>=1.48
 PyYAML>=3.1.0
 sh

stackalytics/processor/default_data_processor.py

@@ -17,15 +17,21 @@ import collections
 import hashlib
 import json
 
+from github import MainClass
+from oslo.config import cfg
+import re
 import six
 
 from stackalytics.openstack.common import log as logging
 from stackalytics.processor import normalizer
+from stackalytics.processor import rcs
 from stackalytics.processor import user_processor
 from stackalytics.processor import utils
 
 LOG = logging.getLogger(__name__)
 
+GITHUB_URI_PREFIX = r'^github:\/\/'
+
 
 def _check_default_data_change(runtime_storage_inst, default_data):
     h = hashlib.new('sha1')
@@ -42,39 +48,76 @@ def _check_default_data_change(runtime_storage_inst, default_data):
     return True
 
 
-def _retrieve_project_list_from_gerrit(project_sources, git_base_uri, gerrit):
+def _retrieve_project_list_from_sources(project_sources):
+    for project_source in project_sources:
+        uri = project_source.get('uri') or cfg.CONF.review_uri
+        repo_iterator = []
+        if re.search(rcs.GERRIT_URI_PREFIX, uri):
+            repo_iterator = _retrieve_project_list_from_gerrit(project_source)
+        elif re.search(GITHUB_URI_PREFIX, uri):
+            repo_iterator = _retrieve_project_list_from_github(project_source)
+
+        exclude = set(project_source.get('exclude', []))
+        for repo in repo_iterator:
+            if repo['module'] not in exclude:
+                yield repo
+
+
+def _retrieve_project_list_from_gerrit(project_source):
     LOG.info('Retrieving project list from Gerrit')
     try:
+        gerrit = rcs.Gerrit(None, project_source['uri'])
+        key_filename = (project_source.get('ssh_key_filename') or
+                        cfg.CONF.ssh_key_filename)
+        username = project_source.get('ssh_username') or cfg.CONF.ssh_username
+        gerrit.setup(key_filename=key_filename, username=username)
+
         project_list = gerrit.get_project_list()
     except Exception as e:
         LOG.exception(e)
         LOG.warn('Fail to retrieve list of projects. Keep it unmodified')
-        return False
+        return
 
-    repos = []
-    for project_source in project_sources:
     organization = project_source['organization']
     LOG.debug('Get list of projects for organization %s', organization)
-        git_repos = [
+    git_repos = [f for f in project_list if f.startswith(organization + "/")]
-            f for f in project_list if f.startswith(organization + "/")]
 
-        exclude = set(project_source.get('exclude', []))
+    git_base_uri = project_source.get('git_base_uri') or cfg.CONF.git_base_uri
 
     for repo in git_repos:
         (org, name) = repo.split('/')
-            if name not in exclude:
+        repo_uri = '%(git_base_uri)s/%(repo)s.git' % dict(
-                url = '%(git_base_uri)s/%(repo)s.git' % dict(
             git_base_uri=git_base_uri, repo=repo)
-                r = {
+        yield {
             'branches': ['master'],
             'module': name,
             'organization': org,
-                    'uri': url,
+            'uri': repo_uri,
+            'releases': []
+        }
+
+
+def _retrieve_project_list_from_github(project_source):
+    LOG.info('Retrieving project list from GitHub')
+    github = MainClass.Github(timeout=60)
+
+    organization = project_source['organization']
+    LOG.debug('Get list of projects for organization %s', organization)
+    try:
+        github_repos = github.get_organization(organization).get_repos()
+    except Exception as e:
+        LOG.exception(e)
+        LOG.warn('Fail to retrieve list of projects. Keep it unmodified')
+        return
+
+    for repo in github_repos:
+        yield {
+            'branches': ['master'],
+            'module': repo.name.lower(),
+            'organization': organization,
+            'uri': repo.git_url,
             'releases': []
         }
-                repos.append(r)
-                LOG.debug('Project is added to default data: %s', r)
-    return repos
 
 
 def _create_module_groups_for_project_sources(project_sources, repos):
@@ -96,12 +139,12 @@ def _create_module_groups_for_project_sources(project_sources, repos):
     return module_groups
 
 
-def _update_project_list(default_data, git_base_uri, gerrit):
+def _update_project_list(default_data):
 
     configured_repos = set([r['uri'] for r in default_data['repos']])
 
-    repos = _retrieve_project_list_from_gerrit(
+    repos = _retrieve_project_list_from_sources(
-        default_data['project_sources'], git_base_uri, gerrit)
+        default_data['project_sources'])
     if repos:
         default_data['repos'] += [r for r in repos
                                   if r['uri'] not in configured_repos]
@@ -198,12 +241,11 @@ def _store_default_data(runtime_storage_inst, default_data):
             runtime_storage_inst.set_by_key(key, value)
 
 
-def process(runtime_storage_inst, default_data,
+def process(runtime_storage_inst, default_data, driverlog_data_uri):
-            git_base_uri, gerrit, driverlog_data_uri):
     LOG.debug('Process default data')
 
     if 'project_sources' in default_data:
-        _update_project_list(default_data, git_base_uri, gerrit)
+        _update_project_list(default_data)
 
     _update_with_driverlog_data(default_data, driverlog_data_uri)
 

stackalytics/processor/main.py

@@ -333,8 +333,6 @@ def main():
 
     default_data_processor.process(runtime_storage_inst,
                                    default_data,
-                                   cfg.CONF.git_base_uri,
-                                   gerrit,
                                    cfg.CONF.driverlog_data_uri)
 
     process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)

tests/unit/test_default_data_processor.py

@@ -72,11 +72,11 @@ class TestDefaultDataProcessor(testtools.TestCase):
                      'organization': 'stackforge'},
                 ],
                 'project_sources': [{'organization': 'openstack',
-                                     'module_group_name': 'OpenStack'}],
+                                     'uri': 'gerrit://'}],
                 'module_groups': [],
             }
 
-            default_data_processor._update_project_list(dd, None, None)
+            default_data_processor._update_project_list(dd)
 
             self.assertEqual(3, len(dd['repos']))
             self.assertIn('qa', set([r['module'] for r in dd['repos']]))
@@ -85,7 +85,7 @@ class TestDefaultDataProcessor(testtools.TestCase):
 
             self.assertEqual(2, len(dd['module_groups']))
             self.assertIn({'id': 'openstack',
-                           'module_group_name': 'OpenStack',
+                           'module_group_name': 'openstack',
                            'modules': ['qa', 'nova'],
                            'tag': 'organization'}, dd['module_groups'])
             self.assertIn({'id': 'stackforge',