x/stackalytics
Code Issues Proposed changes

Fixed review control sustem polling

Browse Source 
* Stop grabbing reviews if id is below last_id (was equal)
* Fixed command for last_id retrieval

* Also fixed blueprint match pattern in VCS

Change-Id: I9fd70db2925ef0b68635c14a68778690b6e6a60e
This commit is contained in:
Ilya Shakhat 2013-09-13 17:07:13 +04:00
parent 1f132ae73d
commit ad0d38066e
5 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
etc/test_default_data.json
View File

@ -81,6 +81,7 @@
],
"uri": "git://github.com/openstack/glance.git",
"module": "glance",
"organization": "openstack",
"project_type": "openstack"
},
{
@ -88,12 +89,14 @@
"module": "python-glanceclient",
"project_group": "core",
"project_type": "openstack",
"organization": "openstack",
"uri": "git://github.com/openstack/python-glanceclient.git"
},
{
"branches": ["master"],
"module": "stackalytics",
"project_type": "stackforge",
"organization": "stackforge",
"uri": "git://github.com/stackforge/stackalytics.git"
}
],

1
stackalytics/processor/default_data_processor.py
View File

@ -71,6 +71,7 @@ def _retrieve_project_list(runtime_storage_inst, project_sources):
r = {
'branches': ['master'],
'module': repo_name,
'organization': organization,
'project_type': project_source['project_type'],
'project_group': project_source['project_group'],
'uri': repo_uri,

38
stackalytics/processor/rcs.py
View File

@ -75,13 +75,13 @@ class Gerrit(Rcs):
username=self.username)
LOG.debug('Successfully connected to Gerrit')
def _get_cmd(self, project_organization, module, branch, sort_key,
is_open):
def _get_cmd(self, project_organization, module, branch, sort_key=None,
is_open=False, limit=PAGE_LIMIT):
cmd = ('gerrit query --all-approvals --patch-sets --format JSON '
'project:\'%(ogn)s/%(module)s\' branch:%(branch)s '
'limit:%(limit)s' %
{'ogn': project_organization, 'module': module,
'branch': branch, 'limit': PAGE_LIMIT})
'branch': branch, 'limit': limit})
if is_open:
cmd += ' is:open'
if sort_key:
@ -104,7 +104,7 @@ class Gerrit(Rcs):
if 'sortKey' in review:
sort_key = int(review['sortKey'], 16)
if sort_key == last_id:
if sort_key <= last_id:
proceed = False
break
@ -116,41 +116,34 @@ class Gerrit(Rcs):
break
def log(self, branch, last_id):
match = re.search(r'([^\/]+)/([^\/]+)\.git$', self.repo['uri'])
if not match:
LOG.error('Invalid repo uri: %s', self.repo['uri'])
project_organization = match.group(1)
module = match.group(2)
self._connect()
# poll new reviews from the top down to last_id
LOG.debug('Poll new reviews')
for review in self._poll_reviews(project_organization, module, branch,
LOG.debug('Poll new reviews for module: %s', self.repo['module'])
for review in self._poll_reviews(self.repo['organization'],
self.repo['module'], branch,
last_id=last_id):
yield review
# poll open reviews from last_id down to bottom
LOG.debug('Poll open reviews')
LOG.debug('Poll open reviews for module: %s', self.repo['module'])
start_id = None
if last_id:
start_id = last_id + 1 # include the last review into query
for review in self._poll_reviews(project_organization, module, branch,
for review in self._poll_reviews(self.repo['organization'],
self.repo['module'], branch,
start_id=start_id, is_open=True):
yield review
self.client.close()
def get_last_id(self, branch):
module = self.repo['module']
LOG.debug('Get last id for module %s', module)
self._connect()
LOG.debug('Get last id for module: %s', self.repo['module'])
cmd = ('gerrit query --all-approvals --patch-sets --format JSON '
'%(module)s branch:%(branch)s limit:1' %
{'module': module, 'branch': branch})
cmd = self._get_cmd(self.repo['organization'], self.repo['module'],
branch, limit=1)
LOG.debug('Executing command: %s', cmd)
stdin, stdout, stderr = self.client.exec_command(cmd)
last_id = None
for line in stdout:
@ -161,7 +154,8 @@ class Gerrit(Rcs):
self.client.close()
LOG.debug('Last id for module %s is %s', module, last_id)
LOG.debug('Module %(module)s last id is %(id)s',
{'module': self.repo['module'], 'id': last_id})
return last_id

2
stackalytics/processor/vcs.py
View File

@ -69,7 +69,7 @@ GIT_LOG_PATTERN = re.compile(''.join([(r[0] + ':(.*?)\n')
MESSAGE_PATTERNS = {
'bug_id': re.compile(r'bug[\s#:]*(?P<id>\d+)', re.IGNORECASE),
'blueprint_id': re.compile(r'\b(?:blueprint|bp)\b[ \t]*[#:]?[ \t]*'
r'(?P<id>\S+)', re.IGNORECASE),
r'(?P<id>[a-z0-9-]+)', re.IGNORECASE),
'change_id': re.compile('Change-Id: (?P<id>I[0-9a-f]{40})', re.IGNORECASE),
}

6
tests/unit/test_vcs.py
View File

@ -44,7 +44,7 @@ date:1369119386
author_name:Akihiro MOTOKI
author_email:motoki@da.jp.nec.com
subject:Remove class-based import in the code repo
message:Fixes bug 1167901
message:Fixes bug 1167901.
This commit also removes backslashes for line break.
@ -83,7 +83,7 @@ date:1369831203
author_name:John Doe
author_email:john.doe@dreamhost.com
subject:add readme for 2.2.2
message: bp fix-me
message: implements blueprint fix-me.
Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7
diff_stat:
@ -122,6 +122,8 @@ diff_stat:
self.assertEquals(0, commits[3]['files_changed'])
self.assertEquals(0, commits[3]['lines_added'])
self.assertEquals(0, commits[3]['lines_deleted'])
self.assertEquals(set(['fix-me']),
set(commits[3]['blueprint_id']))
self.assertEquals(0, commits[4]['files_changed'])
self.assertEquals(0, commits[4]['lines_added'])