From df6753fe6608facbbae57abcd0c42ebbae8fd3e4 Mon Sep 17 00:00:00 2001 From: Ken'ichi Ohmichi Date: Mon, 13 Mar 2017 13:08:27 -0700 Subject: [PATCH] Add fetching_user_source config option Launchpad manages many information related to OpenStack at this time, but the query takes much time for the processor. In addition, stackalytics can be useful for OSS which are not managed with the launchpad. So this patch adds a config option fetching_source so that users can select fetching source. The default option is 'launchpad', so this patch doesn't affect at all on the default option. Change-Id: I4b7db745c36545e8897132cf6ccbb1c3fd6f3c07 --- doc/source/tools/stackalytics-processor.txt | 3 +++ etc/stackalytics.conf | 4 ++++ stackalytics/processor/config.py | 3 +++ stackalytics/processor/record_processor.py | 19 ++++++++++++------- .../tests/unit/test_record_processor.py | 5 +++++ 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/doc/source/tools/stackalytics-processor.txt b/doc/source/tools/stackalytics-processor.txt index e1180acc0..b747b7e01 100644 --- a/doc/source/tools/stackalytics-processor.txt +++ b/doc/source/tools/stackalytics-processor.txt @@ -3,6 +3,7 @@ usage: stackalytics-processor [-h] [--config-dir DIR] [--config-file PATH] [--days_to_update_members DAYS_TO_UPDATE_MEMBERS] [--debug] [--default-data-uri DEFAULT_DATA_URI] [--driverlog-data-uri DRIVERLOG_DATA_URI] + [--fetching-user-source FETCHING_USER_SOURCE] [--gerrit-retry GERRIT_RETRY] [--git-base-uri GIT_BASE_URI] [--log-config-append PATH] @@ -44,6 +45,8 @@ optional arguments: file:///path/to/default_data.json --driverlog-data-uri DRIVERLOG_DATA_URI URI for default data + --fetching-user-source FETCHING_USER_SOURCE + Source for fetching user profiles --gerrit-retry GERRIT_RETRY How many times to retry after Gerrit errors --git-base-uri GIT_BASE_URI diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf index 321b5d2a5..ef8f51236 100644 --- a/etc/stackalytics.conf +++ b/etc/stackalytics.conf @@ -147,6 +147,10 @@ # URI of translation team data (string value) #translation_team_uri = https://git.openstack.org/cgit/openstack/i18n/plain/tools/zanata/translation_team.yaml +# Source for fetching user profiles (string value) +# Allowed values: launchpad, +#fetching_user_source = launchpad + # How many member profiles to look ahead after the last (integer value) #members_look_ahead = 250 diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py index 18e922554..c7b630bd9 100644 --- a/stackalytics/processor/config.py +++ b/stackalytics/processor/config.py @@ -54,6 +54,9 @@ PROCESSOR_OPTS = [ default='https://git.openstack.org/cgit/openstack/i18n/' 'plain/tools/zanata/translation_team.yaml', help='URI of translation team data'), + cfg.StrOpt("fetching-user-source", default='launchpad', + choices=['launchpad', ''], + help="Source for fetching user profiles"), cfg.IntOpt('members-look-ahead', default=250, help='How many member profiles to look ahead after the last'), cfg.IntOpt('read-timeout', default=120, diff --git a/stackalytics/processor/record_processor.py b/stackalytics/processor/record_processor.py index 9f139dd4d..f801cba9f 100644 --- a/stackalytics/processor/record_processor.py +++ b/stackalytics/processor/record_processor.py @@ -19,6 +19,7 @@ import copy import functools import time +from oslo_config import cfg from oslo_log import log as logging import six @@ -27,6 +28,7 @@ from stackalytics.processor import user_processor from stackalytics.processor import utils +CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -232,6 +234,9 @@ class RecordProcessor(object): self.runtime_storage_inst, u) return merged_user + def _need_to_fetch_launchpad(self): + return CONF.fetching_user_source == 'launchpad' + def update_user(self, record): email = record.get('author_email') user_e = user_processor.load_user( @@ -239,8 +244,8 @@ class RecordProcessor(object): user_name = record.get('author_name') launchpad_id = record.get('launchpad_id') - if (email and (not user_e) and (not launchpad_id) and - (not user_e.get('launchpad_id'))): + if (self._need_to_fetch_launchpad() and email and (not user_e) and + (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP launchpad_id, lp_user_name = self._get_lp_info(email) if lp_user_name: @@ -250,8 +255,8 @@ class RecordProcessor(object): if gerrit_id: user_g = user_processor.load_user( self.runtime_storage_inst, gerrit_id=gerrit_id) or {} - if ((not user_g) and (not launchpad_id) and - (not user_e.get('launchpad_id'))): + if (self._need_to_fetch_launchpad() and (not user_g) and + (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP guessed_lp_id = gerrit_id lp_user_name = self._get_lp_user_name(guessed_lp_id) @@ -264,8 +269,8 @@ class RecordProcessor(object): if zanata_id: user_z = user_processor.load_user( self.runtime_storage_inst, zanata_id=zanata_id) or {} - if ((not user_z) and (not launchpad_id) and - (not user_e.get('launchpad_id'))): + if (self._need_to_fetch_launchpad() and (not user_z) and + (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP guessed_lp_id = zanata_id user_name = self._get_lp_user_name(guessed_lp_id) @@ -290,7 +295,7 @@ class RecordProcessor(object): [user_e, user_l, user_g, user_z, user]) else: # create new - if not user_name: + if (self._need_to_fetch_launchpad() and not user_name): user_name = self._get_lp_user_name(launchpad_id) if user_name: user['user_name'] = user_name diff --git a/stackalytics/tests/unit/test_record_processor.py b/stackalytics/tests/unit/test_record_processor.py index 91d33cca7..ef339aa33 100644 --- a/stackalytics/tests/unit/test_record_processor.py +++ b/stackalytics/tests/unit/test_record_processor.py @@ -16,15 +16,19 @@ import time import mock +from oslo_config import cfg import six import testtools +from stackalytics.processor import config from stackalytics.processor import record_processor from stackalytics.processor import runtime_storage from stackalytics.processor import user_processor from stackalytics.processor import utils +CONF = cfg.CONF + RELEASES = [ { 'release_name': 'prehistory', @@ -67,6 +71,7 @@ class TestRecordProcessor(testtools.TestCase): self.lp_profile_by_email = ( self.lp_profile_by_email_patch.start()) self.lp_profile_by_email.return_value = None + CONF.register_opts(config.CONNECTION_OPTS + config.PROCESSOR_OPTS) def tearDown(self): super(TestRecordProcessor, self).tearDown()