Merge "Add fetching_user_source config option"

doc/source/tools/stackalytics-processor.txt

@@ -3,6 +3,7 @@ usage: stackalytics-processor [-h] [--config-dir DIR] [--config-file PATH]
                               [--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
                               [--debug] [--default-data-uri DEFAULT_DATA_URI]
                               [--driverlog-data-uri DRIVERLOG_DATA_URI]
+                              [--fetching-user-source FETCHING_USER_SOURCE]
                               [--gerrit-retry GERRIT_RETRY]
                               [--git-base-uri GIT_BASE_URI]
                               [--log-config-append PATH]
@@ -44,6 +45,8 @@ optional arguments:
                         file:///path/to/default_data.json
   --driverlog-data-uri DRIVERLOG_DATA_URI
                         URI for default data
+  --fetching-user-source FETCHING_USER_SOURCE
+                        Source for fetching user profiles
   --gerrit-retry GERRIT_RETRY
                         How many times to retry after Gerrit errors
   --git-base-uri GIT_BASE_URI

etc/stackalytics.conf

@@ -176,6 +176,10 @@
 # URI of translation team data (string value)
 #translation_team_uri = https://git.openstack.org/cgit/openstack/i18n/plain/tools/zanata/translation_team.yaml
 
+# Source for fetching user profiles (string value)
+# Allowed values: launchpad, <None>
+#fetching_user_source = launchpad
+
 # How many member profiles to look ahead after the last (integer value)
 #members_look_ahead = 250
 

stackalytics/processor/config.py

@@ -54,6 +54,9 @@ PROCESSOR_OPTS = [
                default='https://git.openstack.org/cgit/openstack/i18n/'
                        'plain/tools/zanata/translation_team.yaml',
                help='URI of translation team data'),
+    cfg.StrOpt("fetching-user-source", default='launchpad',
+               choices=['launchpad', '<None>'],
+               help="Source for fetching user profiles"),
     cfg.IntOpt('members-look-ahead', default=250,
                help='How many member profiles to look ahead after the last'),
     cfg.IntOpt('read-timeout', default=120,

stackalytics/processor/record_processor.py

@@ -19,6 +19,7 @@ import copy
 import functools
 import time
 
+from oslo_config import cfg
 from oslo_log import log as logging
 import six
 
@@ -27,6 +28,7 @@ from stackalytics.processor import user_processor
 from stackalytics.processor import utils
 
 
+CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
 
@@ -232,6 +234,9 @@ class RecordProcessor(object):
                         self.runtime_storage_inst, u)
         return merged_user
 
+    def _need_to_fetch_launchpad(self):
+        return CONF.fetching_user_source == 'launchpad'
+
     def update_user(self, record):
         email = record.get('author_email')
         user_e = user_processor.load_user(
@@ -239,8 +244,8 @@ class RecordProcessor(object):
 
         user_name = record.get('author_name')
         launchpad_id = record.get('launchpad_id')
-        if (email and (not user_e) and (not launchpad_id) and
-                (not user_e.get('launchpad_id'))):
+        if (self._need_to_fetch_launchpad() and email and (not user_e) and
+                (not launchpad_id) and (not user_e.get('launchpad_id'))):
             # query LP
             launchpad_id, lp_user_name = self._get_lp_info(email)
             if lp_user_name:
@@ -250,8 +255,8 @@ class RecordProcessor(object):
         if gerrit_id:
             user_g = user_processor.load_user(
                 self.runtime_storage_inst, gerrit_id=gerrit_id) or {}
-            if ((not user_g) and (not launchpad_id) and
-                    (not user_e.get('launchpad_id'))):
+            if (self._need_to_fetch_launchpad() and (not user_g) and
+                    (not launchpad_id) and (not user_e.get('launchpad_id'))):
                 # query LP
                 guessed_lp_id = gerrit_id
                 lp_user_name = self._get_lp_user_name(guessed_lp_id)
@@ -264,8 +269,8 @@ class RecordProcessor(object):
         if zanata_id:
             user_z = user_processor.load_user(
                 self.runtime_storage_inst, zanata_id=zanata_id) or {}
-            if ((not user_z) and (not launchpad_id) and
-                    (not user_e.get('launchpad_id'))):
+            if (self._need_to_fetch_launchpad() and (not user_z) and
+                    (not launchpad_id) and (not user_e.get('launchpad_id'))):
                 # query LP
                 guessed_lp_id = zanata_id
                 user_name = self._get_lp_user_name(guessed_lp_id)
@@ -290,7 +295,7 @@ class RecordProcessor(object):
                     [user_e, user_l, user_g, user_z, user])
             else:
                 # create new
-                if not user_name:
+                if (self._need_to_fetch_launchpad() and not user_name):
                     user_name = self._get_lp_user_name(launchpad_id)
                     if user_name:
                         user['user_name'] = user_name

stackalytics/tests/unit/test_record_processor.py

@@ -16,15 +16,19 @@
 import time
 
 import mock
+from oslo_config import cfg
 import six
 import testtools
 
+from stackalytics.processor import config
 from stackalytics.processor import record_processor
 from stackalytics.processor import runtime_storage
 from stackalytics.processor import user_processor
 from stackalytics.processor import utils
 
 
+CONF = cfg.CONF
+
 RELEASES = [
     {
         'release_name': 'prehistory',
@@ -67,6 +71,7 @@ class TestRecordProcessor(testtools.TestCase):
         self.lp_profile_by_email = (
             self.lp_profile_by_email_patch.start())
         self.lp_profile_by_email.return_value = None
+        CONF.register_opts(config.CONNECTION_OPTS + config.PROCESSOR_OPTS)
 
     def tearDown(self):
         super(TestRecordProcessor, self).tearDown()