From ad0d38066edfbb0b662a6b2281f18f801eb9c488 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Fri, 13 Sep 2013 17:07:13 +0400 Subject: [PATCH] Fixed review control sustem polling * Stop grabbing reviews if id is below last_id (was equal) * Fixed command for last_id retrieval * Also fixed blueprint match pattern in VCS Change-Id: I9fd70db2925ef0b68635c14a68778690b6e6a60e --- etc/test_default_data.json | 3 ++ .../processor/default_data_processor.py | 1 + stackalytics/processor/rcs.py | 38 ++++++++----------- stackalytics/processor/vcs.py | 2 +- tests/unit/test_vcs.py | 6 ++- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/etc/test_default_data.json b/etc/test_default_data.json index 1bed77075..f364ed23b 100644 --- a/etc/test_default_data.json +++ b/etc/test_default_data.json @@ -81,6 +81,7 @@ ], "uri": "git://github.com/openstack/glance.git", "module": "glance", + "organization": "openstack", "project_type": "openstack" }, { @@ -88,12 +89,14 @@ "module": "python-glanceclient", "project_group": "core", "project_type": "openstack", + "organization": "openstack", "uri": "git://github.com/openstack/python-glanceclient.git" }, { "branches": ["master"], "module": "stackalytics", "project_type": "stackforge", + "organization": "stackforge", "uri": "git://github.com/stackforge/stackalytics.git" } ], diff --git a/stackalytics/processor/default_data_processor.py b/stackalytics/processor/default_data_processor.py index 4a5894fae..3492d49e7 100644 --- a/stackalytics/processor/default_data_processor.py +++ b/stackalytics/processor/default_data_processor.py @@ -71,6 +71,7 @@ def _retrieve_project_list(runtime_storage_inst, project_sources): r = { 'branches': ['master'], 'module': repo_name, + 'organization': organization, 'project_type': project_source['project_type'], 'project_group': project_source['project_group'], 'uri': repo_uri, diff --git a/stackalytics/processor/rcs.py b/stackalytics/processor/rcs.py index b95c7741e..2606e69ac 100644 --- a/stackalytics/processor/rcs.py +++ b/stackalytics/processor/rcs.py @@ -75,13 +75,13 @@ class Gerrit(Rcs): username=self.username) LOG.debug('Successfully connected to Gerrit') - def _get_cmd(self, project_organization, module, branch, sort_key, - is_open): + def _get_cmd(self, project_organization, module, branch, sort_key=None, + is_open=False, limit=PAGE_LIMIT): cmd = ('gerrit query --all-approvals --patch-sets --format JSON ' 'project:\'%(ogn)s/%(module)s\' branch:%(branch)s ' 'limit:%(limit)s' % {'ogn': project_organization, 'module': module, - 'branch': branch, 'limit': PAGE_LIMIT}) + 'branch': branch, 'limit': limit}) if is_open: cmd += ' is:open' if sort_key: @@ -104,7 +104,7 @@ class Gerrit(Rcs): if 'sortKey' in review: sort_key = int(review['sortKey'], 16) - if sort_key == last_id: + if sort_key <= last_id: proceed = False break @@ -116,41 +116,34 @@ class Gerrit(Rcs): break def log(self, branch, last_id): - match = re.search(r'([^\/]+)/([^\/]+)\.git$', self.repo['uri']) - if not match: - LOG.error('Invalid repo uri: %s', self.repo['uri']) - project_organization = match.group(1) - module = match.group(2) - self._connect() # poll new reviews from the top down to last_id - LOG.debug('Poll new reviews') - for review in self._poll_reviews(project_organization, module, branch, + LOG.debug('Poll new reviews for module: %s', self.repo['module']) + for review in self._poll_reviews(self.repo['organization'], + self.repo['module'], branch, last_id=last_id): yield review # poll open reviews from last_id down to bottom - LOG.debug('Poll open reviews') + LOG.debug('Poll open reviews for module: %s', self.repo['module']) start_id = None if last_id: start_id = last_id + 1 # include the last review into query - for review in self._poll_reviews(project_organization, module, branch, + for review in self._poll_reviews(self.repo['organization'], + self.repo['module'], branch, start_id=start_id, is_open=True): yield review self.client.close() def get_last_id(self, branch): - module = self.repo['module'] - LOG.debug('Get last id for module %s', module) - self._connect() + LOG.debug('Get last id for module: %s', self.repo['module']) - cmd = ('gerrit query --all-approvals --patch-sets --format JSON ' - '%(module)s branch:%(branch)s limit:1' % - {'module': module, 'branch': branch}) - + cmd = self._get_cmd(self.repo['organization'], self.repo['module'], + branch, limit=1) + LOG.debug('Executing command: %s', cmd) stdin, stdout, stderr = self.client.exec_command(cmd) last_id = None for line in stdout: @@ -161,7 +154,8 @@ class Gerrit(Rcs): self.client.close() - LOG.debug('Last id for module %s is %s', module, last_id) + LOG.debug('Module %(module)s last id is %(id)s', + {'module': self.repo['module'], 'id': last_id}) return last_id diff --git a/stackalytics/processor/vcs.py b/stackalytics/processor/vcs.py index d9c0761ab..26c273a00 100644 --- a/stackalytics/processor/vcs.py +++ b/stackalytics/processor/vcs.py @@ -69,7 +69,7 @@ GIT_LOG_PATTERN = re.compile(''.join([(r[0] + ':(.*?)\n') MESSAGE_PATTERNS = { 'bug_id': re.compile(r'bug[\s#:]*(?P\d+)', re.IGNORECASE), 'blueprint_id': re.compile(r'\b(?:blueprint|bp)\b[ \t]*[#:]?[ \t]*' - r'(?P\S+)', re.IGNORECASE), + r'(?P[a-z0-9-]+)', re.IGNORECASE), 'change_id': re.compile('Change-Id: (?PI[0-9a-f]{40})', re.IGNORECASE), } diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 6105ed0fa..4954e9567 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -44,7 +44,7 @@ date:1369119386 author_name:Akihiro MOTOKI author_email:motoki@da.jp.nec.com subject:Remove class-based import in the code repo -message:Fixes bug 1167901 +message:Fixes bug 1167901. This commit also removes backslashes for line break. @@ -83,7 +83,7 @@ date:1369831203 author_name:John Doe author_email:john.doe@dreamhost.com subject:add readme for 2.2.2 -message: bp fix-me +message: implements blueprint fix-me. Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7 diff_stat: @@ -122,6 +122,8 @@ diff_stat: self.assertEquals(0, commits[3]['files_changed']) self.assertEquals(0, commits[3]['lines_added']) self.assertEquals(0, commits[3]['lines_deleted']) + self.assertEquals(set(['fix-me']), + set(commits[3]['blueprint_id'])) self.assertEquals(0, commits[4]['files_changed']) self.assertEquals(0, commits[4]['lines_added'])