x/driverlog
Code Issues Proposed changes

Optimize communication with Gerrit

Browse Source 
Also:
1. Look over comments only, do not use votes (because some drivers are
   executed by jenkins)
2. For master branch assign release based on date

Change-Id: I75bb29c8d8cf798a882e64953c784e01fd57a95e
This commit is contained in:
Ilya Shakhat 2015-09-30 19:12:34 +03:00
parent 48b982e4a7
commit b6168a8ab6
6 changed files with 85 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
driverlog/processor/main.py
View File

@ -28,18 +28,6 @@ from driverlog.processor import utils
LOG = logging.getLogger(__name__)
def _find_vote(review, ci_id):
"""Finds vote corresponding to ci_id."""
for approval in (review['currentPatchSet'].get('approvals') or []):
if approval['type'] not in ['Verified', 'VRIF']:
continue
if approval['by'].get('username') == ci_id:
return approval['value'] in ['1', '2']
return None
def find_ci_result(review_iterator, ci):
"""For a given stream of reviews finds result left by specified ci."""
@ -51,9 +39,14 @@ def find_ci_result(review_iterator, ci):
continue
message = comment['message']
prefix = 'Patch Set'
if comment['message'].find(prefix) != 0:
continue # look for special messages only
prefix = 'Patch Set %s:' % review['currentPatchSet']['number']
if comment['message'].find(prefix) != 0:
break # all comments from the latest patch set passed
break # all comments from the latest patch set already parsed
message = message[len(prefix):].strip()
result = None
@ -62,30 +55,36 @@ def find_ci_result(review_iterator, ci):
success_pattern = ci.get('success_pattern')
failure_pattern = ci.get('failure_pattern')
if success_pattern and re.search(success_pattern, message):
result = True
elif failure_pattern and re.search(failure_pattern, message):
result = False
message_lines = (l for l in message.split('\n') if l.strip())
# try to get result from vote
if result is None:
result = _find_vote(review, ci['id'])
line = ''
for line in message_lines:
if success_pattern and re.search(success_pattern, line):
result = True
break
elif failure_pattern and re.search(failure_pattern, line):
result = False
break
if result is not None:
return {
'ci_result': result,
'comment': message,
'comment': line,
'timestamp': comment['timestamp'],
'review_url': review_url,
}
def _get_release_by_branch(releases, branch):
def _get_release_by_branch(record_ts, releases, branch):
"""Translates branch name into release_id."""
release = branch.lower()
if release.find('/') > 0:
return release.split('/')[1]
elif release == 'master':
for r in reversed(releases):
start = utils.date_to_timestamp(r.get('start'))
if record_ts > start:
return r['id'].lower()
return releases[-1]['id'].lower()
@ -94,7 +93,8 @@ def update_drivers(drivers, releases):
Returns True if info was updated
"""
branches = [('stable/' + r['id'].lower()) for r in releases] + ['master']
branches = [('stable/' + r['id'].lower()) for r in releases
if r.get('active')] + ['master']
rcs_inst = rcs.get_rcs(cfg.CONF.review_uri)
rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename,
@ -121,13 +121,17 @@ def update_drivers(drivers, releases):
reviewer=ci_id)
ci_result = find_ci_result(review_iterator, driver['ci'])
if ci_result:
LOG.debug('Found CI result: %s', ci_result)
LOG.info('Found CI result: %s', ci_result)
has_updates = True
key = (project_id, driver['vendor'], driver['name'])
os_version = _get_release_by_branch(releases, branch)
os_version = _get_release_by_branch(ci_result['timestamp'],
releases, branch)
ci_result['ci_tested'] = True
drivers[key]['releases'][os_version] = ci_result
else:
LOG.warn('CI result is not found for driver: %s in branch: %s',
driver['name'], branch)
rcs_inst.close()

7
driverlog/processor/rcs.py
View File

@ -26,6 +26,7 @@ LOG = logging.getLogger(__name__)
DEFAULT_PORT = 29418
GERRIT_URI_PREFIX = r'^gerrit:\/\/'
PAGE_LIMIT = 5
CHUNK_LIMIT = 2
class Rcs(object):
@ -110,7 +111,11 @@ class Gerrit(Rcs):
def log(self, **kwargs):
sort_key = None
while True:
# DriverLog looks for merged CRs reviewed by specified gerrit-id
# it is possible that CR is already merged, but CI didn't vote yet
# that's why we grab PAGE_LIMIT*CHUNK_LIMIT CRs from Gerrit
for i in range(CHUNK_LIMIT):
cmd = self._get_cmd(sort_key, **kwargs)
LOG.debug('Executing command: %s', cmd)
exec_result = self._exec_command(cmd)

14
driverlog/tests/unit/test_data/sample_default_data.json
View File

@ -16,15 +16,19 @@
"releases": [
{
"id": "Havana",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Havana"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Havana",
"start": "2013-Apr-04"
},
{
"id": "Icehouse",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Icehouse"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Icehouse",
"start": "2013-Oct-17"
},
{
"id": "Juno",
"wiki": "https://wiki.openstack.org/wiki/Releases"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Juno",
"start": "2014-Apr-17",
"active": true
}
],
"drivers": [
@ -41,7 +45,9 @@
"wiki": "https://wiki.openstack.org/wiki/Arista-neutron-ml2-driver",
"releases": ["Havana", "Icehouse", "Juno"],
"ci": {
"id": "arista-test"
"id": "arista-test",
"success_pattern": "SUCCESS",
"failure_pattern": "FAILURE"
}
},
{

19
driverlog/tests/unit/test_main.py
View File

@ -37,23 +37,6 @@ class TestMain(testtools.TestCase):
def setUp(self):
super(TestMain, self).setUp()
def test_process_reviews_ci_vote_and_comment(self):
# check that vote and matching comment are found
result = main.find_ci_result([_read_sample_review()],
{'id': 'arista-test'})
self.assertIsNotNone(result, 'CI result should be found')
expected_record = {
'ci_result': True,
'comment': 'Verified+1\n\nArista third party testing PASSED '
'[ https://arista.box.com/s/x8z0 ]',
'timestamp': 1399478047,
'review_url': 'https://review.openstack.org/92468',
}
self.assertEqual(expected_record, result)
def test_process_reviews_ci_only_comments(self):
# check that comment is found and parsed correctly
@ -67,7 +50,7 @@ class TestMain(testtools.TestCase):
expected_record = {
'ci_result': True,
'comment': 'Build succeeded.\n\n- neutron_zuul '
'comment': '- neutron_zuul '
'http://128.107.233.28:8080/job/neutron_zuul/263 : '
'SUCCESS in 18m 52s',
'timestamp': 1399481091,

39
etc/default_data.json
View File

@ -36,51 +36,66 @@
"releases": [
{
"id": "Austin",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Austin"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Austin",
"start": "2010-Jan-01"
},
{
"id": "Bexar",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Bexar"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Bexar",
"start": "2010-Oct-21"
},
{
"id": "Cactus",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Cactus"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Cactus",
"start": "2011-Feb-03"
},
{
"id": "Diablo",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Diablo"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Diablo",
"start": "2011-Apr-15"
},
{
"id": "Essex",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Essex"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Essex",
"start": "2011-Sep-22"
},
{
"id": "Folsom",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Folsom"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Folsom",
"start": "2012-Apr-05"
},
{
"id": "Grizzly",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Grizzly"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Grizzly",
"start": "2012-Oct-04"
},
{
"id": "Havana",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Havana"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Havana",
"start": "2013-Apr-04"
},
{
"id": "Icehouse",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Icehouse"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Icehouse",
"start": "2013-Oct-17"
},
{
"id": "Juno",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Juno"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Juno",
"start": "2014-Apr-17",
"active": true
},
{
"id": "Kilo",
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Kilo"
"wiki": "https://wiki.openstack.org/wiki/ReleaseNotes/Kilo",
"start": "2014-Oct-16",
"active": true
},
{
"id": "Liberty",
"wiki": "https://wiki.openstack.org/wiki/Releases"
"wiki": "https://wiki.openstack.org/wiki/Releases",
"start": "2015-Apr-30",
"active": true
}
],
"drivers": [

14
etc/default_data.schema.json
View File

@ -30,9 +30,15 @@
},
"wiki": {
"type": "string"
},
"active": {
"type": "boolean"
},
"start": {
"$ref": "#/definitions/date_format"
}
},
"required": ["id", "wiki"]
"required": ["id", "wiki", "start"]
}
},
"drivers": {
@ -117,12 +123,16 @@
"type": "string"
}
},
"required": ["id"],
"required": ["id", "success_pattern", "failure_pattern"],
"additionalProperties": false
},
"release_id": {
"type": "string",
"pattern": "^[A-Z][a-z]+$"
},
"date_format": {
"type": ["string"],
"pattern": "^20\\d{2}-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-[0-3]\\d$"
}
}
}