From 97c0d89ea5c8bf5e187e66aac1e9e4f7240eac2e Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Sat, 30 Aug 2014 15:57:30 -0700 Subject: [PATCH] Support paging in queries When querying for a list of changes to sync, use the sortkey to handle multiple pages of query results. By default, Gerrit only returns 500 results. Change-Id: I12049b4dcfa796650099b9ba71377a5bcf3feae0 --- gertty/sync.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/gertty/sync.py b/gertty/sync.py index f1719af..ec3e108 100644 --- a/gertty/sync.py +++ b/gertty/sync.py @@ -154,8 +154,21 @@ class SyncProjectTask(Task): if project.updated: # Allow 4 seconds for request time, etc. query += ' -age:%ss' % (int(math.ceil((now-project.updated).total_seconds())) + 4,) - changes = sync.get('changes/?q=%s' % query) - self.log.debug('Query: %s ' % (query,)) + else: + query += ' status:open' + changes = [] + sortkey = '' + while True: + # We don't actually want to limit to 500, but that's the server-side default, and + # if we don't specify this, we won't get a _more_changes flag. + q = 'changes/?n=500%s&q=%s' % (sortkey, query,) + self.log.debug('Query: %s ' % (q,)) + batch = sync.get(q) + changes += batch + if batch and '_more_changes' in batch[-1]: + sortkey = '&N=%s' % (batch[-1]['_sortkey'],) + else: + break with app.db.getSession() as session: for c in changes: # For now, just sync open changes or changes already