diff --git a/gertty/alembic/versions/725816dc500_add_fetch_ref_column.py b/gertty/alembic/versions/725816dc500_add_fetch_ref_column.py new file mode 100644 index 0000000..723bb25 --- /dev/null +++ b/gertty/alembic/versions/725816dc500_add_fetch_ref_column.py @@ -0,0 +1,41 @@ +"""Add fetch ref column + +Revision ID: 725816dc500 +Revises: 38104b4c1b84 +Create Date: 2014-05-31 14:51:08.078616 + +""" + +# revision identifiers, used by Alembic. +revision = '725816dc500' +down_revision = '38104b4c1b84' + +import warnings + +from alembic import op +import sqlalchemy as sa + +from gertty.dbsupport import sqlite_alter_columns + + +def upgrade(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + op.add_column('revision', sa.Column('fetch_auth', sa.Boolean())) + op.add_column('revision', sa.Column('fetch_ref', sa.String(length=255))) + + conn = op.get_bind() + res = conn.execute("select r.key, r.number, c.number from revision r, change c where r.change_key=c.key") + for (rkey, rnumber, cnumber) in res.fetchall(): + q = sa.text("update revision set fetch_auth=:auth, fetch_ref=:ref where key=:key") + ref = 'refs/changes/%s/%s/%s' % (str(cnumber)[-2:], cnumber, rnumber) + res = conn.execute(q, key=rkey, ref=ref, auth=False) + + sqlite_alter_columns('revision', [ + sa.Column('fetch_auth', sa.Boolean(), nullable=False), + sa.Column('fetch_ref', sa.String(length=255), nullable=False) + ]) + +def downgrade(): + op.drop_column('revision', 'fetch_auth') + op.drop_column('revision', 'fetch_ref') diff --git a/gertty/db.py b/gertty/db.py index f129fae..4b5677b 100644 --- a/gertty/db.py +++ b/gertty/db.py @@ -59,6 +59,8 @@ revision_table = Table( Column('message', Text, nullable=False), Column('commit', String(255), nullable=False), Column('parent', String(255), nullable=False), + Column('fetch_auth', Boolean, nullable=False), + Column('fetch_ref', String(255), nullable=False), ) message_table = Table( 'message', metadata, @@ -212,12 +214,14 @@ class Change(object): return l class Revision(object): - def __init__(self, change, number, message, commit, parent): + def __init__(self, change, number, message, commit, parent, fetch_auth, fetch_ref): self.change_key = change.key self.number = number self.message = message self.commit = commit self.parent = parent + self.fetch_auth = fetch_auth + self.fetch_ref = fetch_ref def createMessage(self, *args, **kw): session = Session.object_session(self) diff --git a/gertty/sync.py b/gertty/sync.py index 171a817..2d13eda 100644 --- a/gertty/sync.py +++ b/gertty/sync.py @@ -246,7 +246,9 @@ class SyncChangeTask(Task): url = sync.app.config.url + change.project.name if 'anonymous http' in remote_revision['fetch']: ref = remote_revision['fetch']['anonymous http']['ref'] + auth = False else: + auth = True ref = remote_revision['fetch']['http']['ref'] url = list(urlparse.urlsplit(url)) url[1] = '%s:%s@%s' % (sync.app.config.username, @@ -257,7 +259,8 @@ class SyncChangeTask(Task): if not revision: revision = change.createRevision(remote_revision['_number'], remote_revision['commit']['message'], remote_commit, - remote_revision['commit']['parents'][0]['commit']) + remote_revision['commit']['parents'][0]['commit'], + auth, ref) new_revision = True # TODO: handle multiple parents parent_revision = session.getRevisionByCommit(revision.parent) @@ -382,6 +385,48 @@ class SyncChangeTask(Task): self.log.debug("git fetch %s %s" % (url, ref)) repo.fetch(url, ref) +class CheckRevisionsTask(Task): + def __repr__(self): + return '' + + def run(self, sync): + app = sync.app + to_fetch = [] + with app.db.getSession() as session: + for project in session.getProjects(): + if not project.open_changes: + continue + repo = app.getRepo(project.name) + for change in project.open_changes: + for revision in change.revisions: + if not (repo.hasCommit(revision.parent) and + repo.hasCommit(revision.commit)): + if revision.fetch_ref: + to_fetch.append((project.name, revision.fetch_ref, revision.fetch_auth)) + for name, ref, auth in to_fetch: + sync.submitTask(FetchRefTask(name, ref, auth, priority=self.priority)) + +class FetchRefTask(Task): + def __init__(self, project_name, ref, auth, priority=NORMAL_PRIORITY): + super(FetchRefTask, self).__init__(priority) + self.project_name = project_name + self.ref = ref + self.auth = auth + + def __repr__(self): + return '' % (self.project_name, self.ref) + + def run(self, sync): + # TODO: handle multiple parents + url = sync.app.config.url + self.project_name + if self.auth: + url = list(urlparse.urlsplit(url)) + url[1] = '%s:%s@%s' % (sync.app.config.username, + sync.app.config.password, url[1]) + url = urlparse.urlunsplit(url) + self.log.debug("git fetch %s %s" % (url, self.ref)) + repo = sync.app.getRepo(self.project_name) + repo.fetch(url, self.ref) class UploadReviewsTask(Task): def __repr__(self): @@ -442,9 +487,10 @@ class Sync(object): self.app = app self.log = logging.getLogger('gertty.sync') self.queue = MultiQueue([HIGH_PRIORITY, NORMAL_PRIORITY, LOW_PRIORITY]) + self.submitTask(UploadReviewsTask(HIGH_PRIORITY)) self.submitTask(SyncProjectListTask(HIGH_PRIORITY)) self.submitTask(SyncSubscribedProjectsTask(HIGH_PRIORITY)) - self.submitTask(UploadReviewsTask(HIGH_PRIORITY)) + self.submitTask(CheckRevisionsTask(LOW_PRIORITY)) self.periodic_thread = threading.Thread(target=self.periodicSync) self.periodic_thread.daemon = True self.periodic_thread.start()