ttygroup/boartty
Code Issues Proposed changes

Add account table

Browse Source 
Also add support for dropping sqlite columns in migrations.

Replace any references to user names with references to account table
entries.

Change-Id: I5a147370f5ee648ddd808f699e80a7778c21d662
This commit is contained in:
James E. Blair 2014-07-24 12:29:15 -07:00
parent 6181e4f9bf
commit 9195a05684
6 changed files with 271 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
gertty/alembic/versions/4cc9c46f9d8b_add_account_table.py Normal file
View File

@ -0,0 +1,73 @@
"""add account table
Revision ID: 4cc9c46f9d8b
Revises: 725816dc500
Create Date: 2014-07-23 16:01:47.462597
"""
# revision identifiers, used by Alembic.
revision = '4cc9c46f9d8b'
down_revision = '725816dc500'
import warnings
from alembic import op
import sqlalchemy as sa
from gertty.dbsupport import sqlite_alter_columns, sqlite_drop_columns
def upgrade():
sqlite_drop_columns('message', ['name'])
sqlite_drop_columns('comment', ['name'])
sqlite_drop_columns('approval', ['name'])
sqlite_drop_columns('change', ['owner'])
op.create_table('account',
sa.Column('key', sa.Integer(), nullable=False),
sa.Column('id', sa.Integer(), index=True, unique=True, nullable=False),
sa.Column('name', sa.String(length=255)),
sa.Column('username', sa.String(length=255)),
sa.Column('email', sa.String(length=255)),
sa.PrimaryKeyConstraint('key')
)
op.create_index(op.f('ix_account_name'), 'account', ['name'], unique=True)
op.create_index(op.f('ix_account_username'), 'account', ['name'], unique=True)
op.create_index(op.f('ix_account_email'), 'account', ['name'], unique=True)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
op.add_column('message', sa.Column('account_key', sa.Integer()))
op.add_column('comment', sa.Column('account_key', sa.Integer()))
op.add_column('approval', sa.Column('account_key', sa.Integer()))
op.add_column('change', sa.Column('account_key', sa.Integer()))
sqlite_alter_columns('message', [
sa.Column('account_key', sa.Integer(), sa.ForeignKey('account.key'))
])
sqlite_alter_columns('comment', [
sa.Column('account_key', sa.Integer(), sa.ForeignKey('account.key'))
])
sqlite_alter_columns('approval', [
sa.Column('account_key', sa.Integer(), sa.ForeignKey('account.key'))
])
sqlite_alter_columns('change', [
sa.Column('account_key', sa.Integer(), sa.ForeignKey('account.key'))
])
op.create_index(op.f('ix_message_account_key'), 'message', ['account_key'], unique=False)
op.create_index(op.f('ix_comment_account_key'), 'comment', ['account_key'], unique=False)
op.create_index(op.f('ix_approval_account_key'), 'approval', ['account_key'], unique=False)
op.create_index(op.f('ix_change_account_key'), 'change', ['account_key'], unique=False)
connection = op.get_bind()
project = sa.sql.table('project', sa.sql.column('updated', sa.DateTime))
connection.execute(project.update().values({'updated':None}))
approval = sa.sql.table('approval', sa.sql.column('pending'))
connection.execute(approval.delete().where(approval.c.pending==False))
def downgrade():
pass

79
gertty/db.py
View File

@ -44,7 +44,7 @@ change_table = Table(
Column('branch', String(255), index=True, nullable=False),
Column('change_id', String(255), index=True, nullable=False),
Column('topic', String(255), index=True),
Column('owner', String(255), index=True),
Column('account_key', Integer, ForeignKey("account.key"), index=True),
Column('subject', Text, nullable=False),
Column('created', DateTime, index=True, nullable=False),
Column('updated', DateTime, index=True, nullable=False),
@ -67,9 +67,9 @@ message_table = Table(
'message', metadata,
Column('key', Integer, primary_key=True),
Column('revision_key', Integer, ForeignKey("revision.key"), index=True),
Column('account_key', Integer, ForeignKey("account.key"), index=True),
Column('id', String(255), index=True), #, unique=True, nullable=False),
Column('created', DateTime, index=True, nullable=False),
Column('name', String(255)),
Column('message', Text, nullable=False),
Column('pending', Boolean, index=True, nullable=False),
)
@ -77,10 +77,10 @@ comment_table = Table(
'comment', metadata,
Column('key', Integer, primary_key=True),
Column('revision_key', Integer, ForeignKey("revision.key"), index=True),
Column('account_key', Integer, ForeignKey("account.key"), index=True),
Column('id', String(255), index=True), #, unique=True, nullable=False),
Column('in_reply_to', String(255)),
Column('created', DateTime, index=True, nullable=False),
Column('name', String(255)),
Column('file', Text, nullable=False),
Column('parent', Boolean, nullable=False),
Column('line', Integer),
@ -106,12 +106,26 @@ approval_table = Table(
'approval', metadata,
Column('key', Integer, primary_key=True),
Column('change_key', Integer, ForeignKey("change.key"), index=True),
Column('name', String(255)),
Column('account_key', Integer, ForeignKey("account.key"), index=True),
Column('category', String(255), nullable=False),
Column('value', Integer, nullable=False),
Column('pending', Boolean, index=True, nullable=False),
)
account_table = Table(
'account', metadata,
Column('key', Integer, primary_key=True),
Column('id', Integer, index=True, unique=True, nullable=False),
Column('name', String(255), index=True),
Column('username', String(255), index=True),
Column('email', String(255), index=True),
)
class Account(object):
def __init__(self, id, name=None, username=None, email=None):
self.id = id
self.name = name
self.username = username
self.email = email
class Project(object):
def __init__(self, name, subscribed=False, description=''):
@ -129,16 +143,16 @@ class Project(object):
return c
class Change(object):
def __init__(self, project, id, number, branch, change_id,
owner, subject, created, updated, status,
def __init__(self, project, id, owner, number, branch,
change_id, subject, created, updated, status,
topic=False, hidden=False, reviewed=False):
self.project_key = project.key
self.account_key = owner.key
self.id = id
self.number = number
self.branch = branch
self.change_id = change_id
self.topic = topic
self.owner = owner
self.subject = subject
self.created = created
self.updated = updated
@ -243,21 +257,21 @@ class Revision(object):
return c
class Message(object):
def __init__(self, revision, id, created, name, message, pending=False):
def __init__(self, revision, id, author, created, message, pending=False):
self.revision_key = revision.key
self.account_key = author.key
self.id = id
self.created = created
self.name = name
self.message = message
self.pending = pending
class Comment(object):
def __init__(self, revision, id, in_reply_to, created, name, file, parent, line, message, pending=False):
def __init__(self, revision, id, author, in_reply_to, created, file, parent, line, message, pending=False):
self.revision_key = revision.key
self.account_key = author.key
self.id = id
self.in_reply_to = in_reply_to
self.created = created
self.name = name
self.file = file
self.parent = parent
self.line = line
@ -278,13 +292,14 @@ class PermittedLabel(object):
self.value = value
class Approval(object):
def __init__(self, change, name, category, value, pending=False):
def __init__(self, change, reviewer, category, value, pending=False):
self.change_key = change.key
self.name = name
self.account_key = reviewer.key
self.category = category
self.value = value
self.pending = pending
mapper(Account, account_table)
mapper(Project, project_table, properties=dict(
changes=relationship(Change, backref='project',
order_by=change_table.c.number),
@ -304,6 +319,7 @@ mapper(Project, project_table, properties=dict(
),
))
mapper(Change, change_table, properties=dict(
owner=relationship(Account),
revisions=relationship(Revision, backref='change',
order_by=revision_table.c.number),
messages=relationship(Message,
@ -333,11 +349,14 @@ mapper(Revision, revision_table, properties=dict(
order_by=(comment_table.c.line,
comment_table.c.created)),
))
mapper(Message, message_table)
mapper(Comment, comment_table)
mapper(Message, message_table, properties=dict(
author=relationship(Account)))
mapper(Comment, comment_table, properties=dict(
author=relationship(Account)))
mapper(Label, label_table)
mapper(PermittedLabel, permitted_label_table)
mapper(Approval, approval_table)
mapper(Approval, approval_table, properties=dict(
reviewer=relationship(Account)))
class Database(object):
def __init__(self, app):
@ -505,8 +524,36 @@ class DatabaseSession(object):
def getPendingMessages(self):
return self.session().query(Message).filter_by(pending=True).all()
def getAccountByID(self, id, name=None, username=None, email=None):
try:
account = self.session().query(Account).filter_by(id=id).one()
except sqlalchemy.orm.exc.NoResultFound:
account = self.createAccount(id)
if name is not None and account.name != name:
account.name = name
if username is not None and account.username != username:
account.username = username
if email is not None and account.email != email:
account.email = email
return account
def getAccountByUsername(self, username):
try:
return self.session().query(Account).filter_by(username=username).one()
except sqlalchemy.orm.exc.NoResultFound:
return None
def getSystemAccount(self):
return self.getAccountByID(0, 'Gerrit Code Review')
def createProject(self, *args, **kw):
o = Project(*args, **kw)
self.session().add(o)
self.session().flush()
return o
def createAccount(self, *args, **kw):
a = Account(*args, **kw)
self.session().add(a)
self.session().flush()
return a

67
gertty/dbsupport.py
View File

@ -103,3 +103,70 @@ def sqlite_alter_columns(table_name, column_defs):
# (re-)create indexes
for index in indexes:
op.create_index(op.f(index[0]), index[1], index[2], unique=index[3])
def sqlite_drop_columns(table_name, drop_columns):
"""Implement drop columns for SQLite.
The DROP COLUMN command isn't supported by SQLite specification.
Instead of calling DROP COLUMN it uses the following workaround:
* create temp table '{table_name}_{rand_uuid}', without
dropped columns;
* copy all data to the temp table;
* drop old table;
* rename temp table to the old table name.
"""
connection = op.get_bind()
meta = sqlalchemy.MetaData(bind=connection)
meta.reflect()
# construct lists of all columns and their names
old_columns = []
new_columns = []
column_names = []
indexes = []
for column in meta.tables[table_name].columns:
if column.name not in drop_columns:
old_columns.append(column)
column_names.append(column.name)
col_copy = column.copy()
new_columns.append(col_copy)
for key in meta.tables[table_name].foreign_keys:
constraint = key.constraint
con_copy = constraint.copy()
new_columns.append(con_copy)
for index in meta.tables[table_name].indexes:
# If this is a single column index for a dropped column, don't
# copy it.
idx_columns = [col.name for col in index.columns]
if len(idx_columns)==1 and idx_columns[0] in drop_columns:
continue
# Otherwise, recreate the index.
indexes.append((index.name,
table_name,
[col.name for col in index.columns],
index.unique))
# create temp table
tmp_table_name = "%s_%s" % (table_name, six.text_type(uuid.uuid4()))
op.create_table(tmp_table_name, *new_columns)
meta.reflect()
try:
# copy data from the old table to the temp one
sql_select = sqlalchemy.sql.select(old_columns)
connection.execute(sqlalchemy.sql.insert(meta.tables[tmp_table_name])
.from_select(column_names, sql_select))
except Exception:
op.drop_table(tmp_table_name)
raise
# drop the old table and rename temp table to the old table name
op.drop_table(table_name)
op.rename_table(tmp_table_name, table_name)
# (re-)create indexes
for index in indexes:
op.create_index(op.f(index[0]), index[1], index[2], unique=index[3])

68
gertty/sync.py
View File

@ -87,6 +87,19 @@ class Task(object):
self.event.wait(timeout)
return self.succeeded
class SyncOwnAccountTask(Task):
def __repr__(self):
return '<SyncOwnAccountTask>'
def run(self, sync):
app = sync.app
remote = sync.get('accounts/self')
with app.db.getSession() as session:
session.getAccountByID(remote['_account_id'],
remote.get('name'),
remote['username'],
remote.get('email'))
class SyncProjectListTask(Task):
def __repr__(self):
return '<SyncProjectListTask>'
@ -224,16 +237,20 @@ class SyncChangeTask(Task):
fetches = collections.defaultdict(list)
with app.db.getSession() as session:
change = session.getChangeByID(self.change_id)
account = session.getAccountByID(remote_change['owner']['_account_id'],
name=remote_change['owner'].get('name'),
username=remote_change['owner'].get('username'),
email=remote_change['owner'].get('email'))
if not change:
project = session.getProjectByName(remote_change['project'])
created = dateutil.parser.parse(remote_change['created'])
updated = dateutil.parser.parse(remote_change['updated'])
change = project.createChange(remote_change['id'], remote_change['_number'],
change = project.createChange(remote_change['id'], account, remote_change['_number'],
remote_change['branch'], remote_change['change_id'],
remote_change['owner']['name'],
remote_change['subject'], created,
updated, remote_change['status'],
topic=remote_change.get('topic'))
change.owner = account
change.status = remote_change['status']
change.subject = remote_change['subject']
change.updated = dateutil.parser.parse(remote_change['updated'])
@ -272,6 +289,10 @@ class SyncChangeTask(Task):
remote_comments_data = remote_revision['_gertty_remote_comments_data']
for remote_file, remote_comments in remote_comments_data.items():
for remote_comment in remote_comments:
account = session.getAccountByID(remote_comment['author']['_account_id'],
name=remote_comment['author'].get('name'),
username=remote_comment['author'].get('username'),
email=remote_comment['author'].get('email'))
comment = session.getCommentByID(remote_comment['id'])
if not comment:
# Normalize updated -> created
@ -279,27 +300,35 @@ class SyncChangeTask(Task):
parent = False
if remote_comment.get('side', '') == 'PARENT':
parent = True
comment = revision.createComment(remote_comment['id'],
comment = revision.createComment(remote_comment['id'], account,
remote_comment.get('in_reply_to'),
created, remote_comment['author']['name'],
created,
remote_file, parent, remote_comment.get('line'),
remote_comment['message'])
else:
if comment.author != account:
comment.author = account
new_message = False
for remote_message in remote_change.get('messages', []):
if 'author' in remote_message:
account = session.getAccountByID(remote_message['author']['_account_id'],
name=remote_message['author'].get('name'),
username=remote_message['author'].get('username'),
email=remote_message['author'].get('email'))
if account.username != app.config.username:
new_message = True
else:
account = session.getSystemAccount()
message = session.getMessageByID(remote_message['id'])
if not message:
revision = session.getRevisionByNumber(change, remote_message['_revision_number'])
# Normalize date -> created
created = dateutil.parser.parse(remote_message['date'])
if 'author' in remote_message:
author_name = remote_message['author']['name']
if remote_message['author'].get('username') != app.config.username:
new_message = True
else:
author_name = 'Gerrit Code Review'
message = revision.createMessage(remote_message['id'], created,
author_name,
message = revision.createMessage(remote_message['id'], account, created,
remote_message['message'])
else:
if message.author != account:
message.author = account
remote_approval_entries = {}
remote_label_entries = {}
user_voted = False
@ -324,7 +353,8 @@ class SyncChangeTask(Task):
local_approvals = {}
local_labels = {}
for approval in change.approvals:
key = '%s~%s' % (approval.category, approval.name)
self.log.debug(approval.key)
key = '%s~%s' % (approval.category, approval.reviewer.name)
local_approvals[key] = approval
local_approval_keys = set(local_approvals.keys())
for label in change.labels:
@ -340,7 +370,11 @@ class SyncChangeTask(Task):
for key in remote_approval_keys-local_approval_keys:
remote_approval = remote_approval_entries[key]
change.createApproval(remote_approval['name'],
account = session.getAccountByID(remote_approval['_account_id'],
name=remote_approval.get('name'),
username=remote_approval.get('username'),
email=remote_approval.get('email'))
change.createApproval(account,
remote_approval['category'],
remote_approval['value'])
@ -354,6 +388,11 @@ class SyncChangeTask(Task):
local_approval = local_approvals[key]
remote_approval = remote_approval_entries[key]
local_approval.value = remote_approval['value']
# For the side effect of updating account info:
account = session.getAccountByID(remote_approval['_account_id'],
name=remote_approval.get('name'),
username=remote_approval.get('username'),
email=remote_approval.get('email'))
remote_permitted_entries = {}
for remote_label_name, remote_label_values in remote_change.get('permitted_labels', {}).items():
@ -508,6 +547,7 @@ class Sync(object):
self.session = requests.Session()
self.auth = requests.auth.HTTPDigestAuth(
self.app.config.username, self.app.config.password)
self.submitTask(SyncOwnAccountTask(HIGH_PRIORITY))
self.submitTask(UploadReviewsTask(HIGH_PRIORITY))
self.submitTask(SyncProjectListTask(HIGH_PRIORITY))
self.submitTask(SyncSubscribedProjectsTask(HIGH_PRIORITY))

17
gertty/view/change.py
View File

@ -258,7 +258,7 @@ class ChangeMessageBox(mywid.HyperText):
def __init__(self, app, message):
super(ChangeMessageBox, self).__init__(u'')
lines = message.message.split('\n')
text = [('change-message-name', message.name),
text = [('change-message-name', message.author.name),
('change-message-header', ': '+lines.pop(0)),
('change-message-header',
message.created.strftime(' (%Y-%m-%d %H:%M:%S%z)'))]
@ -388,7 +388,7 @@ class ChangeView(urwid.WidgetWrap):
self.change_rest_id = change.id
self.change_id_label.set_text(('change-data', change.change_id))
self.owner_label.set_text(('change-data', change.owner))
self.owner_label.set_text(('change-data', change.owner.name))
self.project_label.set_text(('change-data', change.project.name))
self.branch_label.set_text(('change-data', change.branch))
self.topic_label.set_text(('change-data', change.topic or ''))
@ -413,16 +413,16 @@ class ChangeView(urwid.WidgetWrap):
votes = mywid.Table(approval_headers)
approvals_for_name = {}
for approval in change.approvals:
approvals = approvals_for_name.get(approval.name)
approvals = approvals_for_name.get(approval.reviewer.name)
if not approvals:
approvals = {}
row = []
row.append(urwid.Text(('reviewer-name', approval.name)))
row.append(urwid.Text(('reviewer-name', approval.reviewer.name)))
for i, category in enumerate(categories):
w = urwid.Text(u'', align=urwid.CENTER)
approvals[category] = w
row.append(w)
approvals_for_name[approval.name] = approvals
approvals_for_name[approval.reviewer.name] = approvals
votes.addRow(row)
if str(approval.value) != '0':
if approval.value > 0:
@ -591,6 +591,7 @@ class ChangeView(urwid.WidgetWrap):
def saveReview(self, revision_key, approvals, message):
message_key = None
with self.app.db.getSession() as session:
account = session.getAccountByUsername(self.app.config.username)
revision = session.getRevision(revision_key)
change = revision.change
pending_approvals = {}
@ -604,7 +605,7 @@ class ChangeView(urwid.WidgetWrap):
value = approvals.get(category, 0)
approval = pending_approvals.get(category)
if not approval:
approval = change.createApproval(u'(draft)', category, 0, pending=True)
approval = change.createApproval(account, category, 0, pending=True)
pending_approvals[category] = approval
approval.value = value
pending_message = None
@ -613,9 +614,9 @@ class ChangeView(urwid.WidgetWrap):
pending_message = m
break
if not pending_message:
pending_message = revision.createMessage(None,
pending_message = revision.createMessage(None, account,
datetime.datetime.utcnow(),
u'(draft)', '', pending=True)
'', pending=True)
pending_message.message = message
message_key = pending_message.key
change.reviewed = True

9
gertty/view/diff.py
View File

@ -268,7 +268,7 @@ class DiffView(urwid.WidgetWrap):
if comment.pending:
message = comment.message
else:
message = [('comment-name', comment.name),
message = [('comment-name', comment.author.name),
('comment', u': '+comment.message)]
comment_list.append((comment.key, message))
comment_lists[key] = comment_list
@ -284,7 +284,7 @@ class DiffView(urwid.WidgetWrap):
if comment.pending:
message = comment.message
else:
message = [('comment-name', comment.name),
message = [('comment-name', comment.author.name),
('comment', u': '+comment.message)]
comment_list.append((comment.key, message))
comment_lists[key] = comment_list
@ -537,9 +537,10 @@ class DiffView(urwid.WidgetWrap):
filename = context.old_fn
with self.app.db.getSession() as session:
revision = session.getRevision(revision_key)
comment = revision.createComment(None, None,
account = session.getAccountByUsername(self.app.config.username)
comment = revision.createComment(None, account, None,
datetime.datetime.utcnow(),
None, filename, parent,
filename, parent,
line_num, text, pending=True)
key = comment.key
return key