x/sqlalchemy-migrate
Code Issues Proposed changes

migrate.versioning.schema and schemadiff PEP-8 clean, added to api.rst

Browse Source
This commit is contained in:
jan.dittberner 2009-01-25 21:09:05 +00:00
parent 2a8860478c
commit 0fa7d4103e
3 changed files with 220 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/api.rst
View File

@ -118,3 +118,17 @@ Module :mod:`repository <migrate.versioning.repository>`
.. automodule:: migrate.versioning.repository .. automodule:: migrate.versioning.repository
:members: :members:
:synopsis: SQLAlchemy migrate repository management :synopsis: SQLAlchemy migrate repository management
Module :mod:`schema <migrate.versioning.schema>`
------------------------------------------------
.. automodule:: migrate.versioning.schema
:members:
:synopsis: Database schema management
Module :mod:`schemadiff <migrate.versioning.schemadiff>`
--------------------------------------------------------
.. automodule:: migrate.versioning.schemadiff
:members:
:synopsis: Database schema and model differencing

167
migrate/versioning/schema.py
View File

@ -1,4 +1,8 @@
from sqlalchemy import Table,Column,MetaData,String,Text,Integer,create_engine """
Database schema version management.
"""
from sqlalchemy import Table, Column, MetaData, String, Text, Integer, \
create_engine
from sqlalchemy.sql import and_ from sqlalchemy.sql import and_
from sqlalchemy import exceptions as sa_exceptions from sqlalchemy import exceptions as sa_exceptions
from migrate.versioning.repository import Repository from migrate.versioning.repository import Repository
@ -6,63 +10,70 @@ from migrate.versioning.util import loadModel
from migrate.versioning.version import VerNum from migrate.versioning.version import VerNum
from migrate.versioning import exceptions, genmodel, schemadiff from migrate.versioning import exceptions, genmodel, schemadiff
class ControlledSchema(object): class ControlledSchema(object):
"""A database under version control""" """A database under version control"""
#def __init__(self,engine,repository=None):
def __init__(self,engine,repository): def __init__(self, engine, repository):
if type(repository) is str: if type(repository) is str:
repository=Repository(repository) repository=Repository(repository)
self.engine = engine self.engine = engine
self.repository = repository self.repository = repository
self.meta=MetaData(engine) self.meta=MetaData(engine)
#if self.repository is None:
# self._get_repository()
self._load() self._load()
def __eq__(self,other): def __eq__(self, other):
return (self.repository is other.repository \ return (self.repository is other.repository \
and self.version == other.version) and self.version == other.version)
def _load(self): def _load(self):
"""Load controlled schema version info from DB""" """Load controlled schema version info from DB"""
tname = self.repository.version_table tname = self.repository.version_table
self.meta=MetaData(self.engine) self.meta=MetaData(self.engine)
if not hasattr(self,'table') or self.table is None: if not hasattr(self, 'table') or self.table is None:
try: try:
self.table = Table(tname,self.meta,autoload=True) self.table = Table(tname, self.meta, autoload=True)
except (exceptions.NoSuchTableError): except (exceptions.NoSuchTableError):
raise exceptions.DatabaseNotControlledError(tname) raise exceptions.DatabaseNotControlledError(tname)
# TODO?: verify that the table is correct (# cols, etc.) # TODO?: verify that the table is correct (# cols, etc.)
result = self.engine.execute(self.table.select( result = self.engine.execute(self.table.select(
self.table.c.repository_id == str(self.repository.id)) self.table.c.repository_id == str(self.repository.id)))
)
data = list(result)[0] data = list(result)[0]
# TODO?: exception if row count is bad # TODO?: exception if row count is bad
# TODO: check repository id, exception if incorrect # TODO: check repository id, exception if incorrect
self.version = data['version'] self.version = data['version']
def _get_repository(self): def _get_repository(self):
"""Given a database engine, try to guess the repository""" """
Given a database engine, try to guess the repository.
:raise: :exc:`NotImplementedError`
"""
# TODO: no guessing yet; for now, a repository must be supplied # TODO: no guessing yet; for now, a repository must be supplied
raise NotImplementedError() raise NotImplementedError()
@classmethod @classmethod
def create(cls,engine,repository,version=None): def create(cls, engine, repository, version=None):
"""Declare a database to be under a repository's version control""" """
# Confirm that the version # is valid: positive, integer, exists in repos Declare a database to be under a repository's version control.
"""
# Confirm that the version # is valid: positive, integer,
# exists in repos
if type(repository) is str: if type(repository) is str:
repository=Repository(repository) repository=Repository(repository)
version = cls._validate_version(repository,version) version = cls._validate_version(repository, version)
table=cls._create_table_version(engine,repository,version) table=cls._create_table_version(engine, repository, version)
# TODO: history table # TODO: history table
# Load repository information and return # Load repository information and return
return cls(engine,repository) return cls(engine, repository)
@classmethod @classmethod
def _validate_version(cls,repository,version): def _validate_version(cls, repository, version):
"""Ensures this is a valid version number for this repository """
If invalid, raises cls.InvalidVersionError Ensures this is a valid version number for this repository.
Returns a valid version number
:raises: :exc:`cls.InvalidVersionError` if invalid
:return: valid version number
""" """
if version is None: if version is None:
version=0 version=0
@ -73,98 +84,114 @@ class ControlledSchema(object):
except ValueError: except ValueError:
raise exceptions.InvalidVersionError(version) raise exceptions.InvalidVersionError(version)
return version return version
@classmethod @classmethod
def _create_table_version(cls,engine,repository,version): def _create_table_version(cls, engine, repository, version):
"""Creates the versioning table in a database""" """
Creates the versioning table in a database.
"""
# Create tables # Create tables
tname = repository.version_table tname = repository.version_table
meta = MetaData(engine) meta = MetaData(engine)
try: try:
table = Table(tname,meta, table = Table(
#Column('repository_id',String,primary_key=True), # MySQL needs a length tname, meta,
Column('repository_id',String(255),primary_key=True), Column('repository_id', String(255), primary_key=True),
Column('repository_path',Text), Column('repository_path', Text),
Column('version',Integer), Column('version', Integer), )
)
table.create() table.create()
except (sa_exceptions.ArgumentError,sa_exceptions.SQLError): except (sa_exceptions.ArgumentError, sa_exceptions.SQLError):
# The table already exists, skip creation. # The table already exists, skip creation.
pass pass
#raise exceptions.DatabaseAlreadyControlledError()
# Insert data # Insert data
try: try:
engine.execute(table.insert(),repository_id=repository.id, engine.execute(table.insert(), repository_id=repository.id,
repository_path=repository.path,version=int(version)) repository_path=repository.path,
version=int(version))
except sa_exceptions.IntegrityError: except sa_exceptions.IntegrityError:
# An Entry for this repo already exists. # An Entry for this repo already exists.
raise exceptions.DatabaseAlreadyControlledError() raise exceptions.DatabaseAlreadyControlledError()
return table return table
@classmethod
def compare_model_to_db(cls,engine,model,repository):
"""Compare the current model against the current database."""
@classmethod
def compare_model_to_db(cls, engine, model, repository):
"""
Compare the current model against the current database.
"""
if isinstance(repository, basestring): if isinstance(repository, basestring):
repository=Repository(repository) repository=Repository(repository)
model = loadModel(model) model = loadModel(model)
diff = schemadiff.getDiffOfModelAgainstDatabase(model, engine, excludeTables=[repository.version_table]) diff = schemadiff.getDiffOfModelAgainstDatabase(
model, engine, excludeTables=[repository.version_table])
return diff return diff
@classmethod @classmethod
def create_model(cls,engine,repository,declarative=False): def create_model(cls, engine, repository, declarative=False):
"""Dump the current database as a Python model.""" """
Dump the current database as a Python model.
"""
if isinstance(repository, basestring): if isinstance(repository, basestring):
repository=Repository(repository) repository=Repository(repository)
diff = schemadiff.getDiffOfModelAgainstDatabase(MetaData(), engine, excludeTables=[repository.version_table]) diff = schemadiff.getDiffOfModelAgainstDatabase(
MetaData(), engine, excludeTables=[repository.version_table])
return genmodel.ModelGenerator(diff, declarative).toPython() return genmodel.ModelGenerator(diff, declarative).toPython()
def update_db_from_model(self,model):
"""Modify the database to match the structure of the current Python model."""
def update_db_from_model(self, model):
"""
Modify the database to match the structure of the current Python model.
"""
if isinstance(self.repository, basestring): if isinstance(self.repository, basestring):
self.repository=Repository(self.repository) self.repository=Repository(self.repository)
model = loadModel(model) model = loadModel(model)
diff = schemadiff.getDiffOfModelAgainstDatabase(model, self.engine, excludeTables=[self.repository.version_table]) diff = schemadiff.getDiffOfModelAgainstDatabase(
model, self.engine, excludeTables=[self.repository.version_table])
genmodel.ModelGenerator(diff).applyModel() genmodel.ModelGenerator(diff).applyModel()
update = self.table.update(self.table.c.repository_id == str(self.repository.id)) update = self.table.update(
self.table.c.repository_id == str(self.repository.id))
self.engine.execute(update, version=int(self.repository.latest)) self.engine.execute(update, version=int(self.repository.latest))
def drop(self): def drop(self):
"""Remove version control from a database""" """
Remove version control from a database.
"""
try: try:
self.table.drop() self.table.drop()
except (sa_exceptions.SQLError): except (sa_exceptions.SQLError):
raise exceptions.DatabaseNotControlledError(str(self.table)) raise exceptions.DatabaseNotControlledError(str(self.table))
def _engine_db(self,engine): def _engine_db(self, engine):
"""Returns the database name of an engine - 'postgres','sqlite'...""" """
Returns the database name of an engine - ``postgres``, ``sqlite`` ...
"""
# TODO: This is a bit of a hack... # TODO: This is a bit of a hack...
return str(engine.dialect.__module__).split('.')[-1] return str(engine.dialect.__module__).split('.')[-1]
def changeset(self,version=None): def changeset(self, version=None):
database = self._engine_db(self.engine) database = self._engine_db(self.engine)
start_ver = self.version start_ver = self.version
changeset = self.repository.changeset(database,start_ver,version) changeset = self.repository.changeset(database, start_ver, version)
return changeset return changeset
def runchange(self,ver,change,step): def runchange(self, ver, change, step):
startver = ver startver = ver
endver = ver + step endver = ver + step
# Current database version must be correct! Don't run if corrupt! # Current database version must be correct! Don't run if corrupt!
if self.version != startver: if self.version != startver:
raise exceptions.InvalidVersionError("%s is not %s"%(self.version,startver)) raise exceptions.InvalidVersionError("%s is not %s" % \
(self.version, startver))
# Run the change # Run the change
change.run(self.engine,step) change.run(self.engine, step)
# Update/refresh database version # Update/refresh database version
update = self.table.update(and_(self.table.c.version == int(startver), update = self.table.update(
self.table.c.repository_id == str(self.repository.id))) and_(self.table.c.version == int(startver),
self.table.c.repository_id == str(self.repository.id)))
self.engine.execute(update, version=int(endver)) self.engine.execute(update, version=int(endver))
self._load() self._load()
def upgrade(self,version=None): def upgrade(self, version=None):
"""Upgrade (or downgrade) to a specified version, or latest version""" """
Upgrade (or downgrade) to a specified version, or latest version.
"""
changeset = self.changeset(version) changeset = self.changeset(version)
for ver,change in changeset: for ver, change in changeset:
self.runchange(ver,change,changeset.step) self.runchange(ver, change, changeset.step)

164
migrate/versioning/schemadiff.py
View File

@ -1,44 +1,63 @@
"""
Schema differencing support.
"""
import sqlalchemy import sqlalchemy
def getDiffOfModelAgainstDatabase(model, conn, excludeTables=None): def getDiffOfModelAgainstDatabase(model, conn, excludeTables=None):
''' Return differences of model against database. """
Returned object will evaluate to True if there are differences else False. Return differences of model against database.
'''
:return: object which will evaluate to :keyword:`True` if there \
are differences else :keyword:`False`.
"""
return SchemaDiff(model, conn, excludeTables) return SchemaDiff(model, conn, excludeTables)
def getDiffOfModelAgainstModel(oldmodel, model, conn, excludeTables=None): def getDiffOfModelAgainstModel(oldmodel, model, conn, excludeTables=None):
''' Return differences of model against database. """
Returned object will evaluate to True if there are differences else False. Return differences of model against another model.
'''
:return: object which will evaluate to :keyword:`True` if there \
are differences else :keyword:`False`.
"""
return SchemaDiff(model, conn, excludeTables, oldmodel=oldmodel) return SchemaDiff(model, conn, excludeTables, oldmodel=oldmodel)
class SchemaDiff(object): class SchemaDiff(object):
''' Differences of model against database. ''' """
Differences of model against database.
"""
def __init__(self, model, conn, excludeTables=None, oldmodel=None): def __init__(self, model, conn, excludeTables=None, oldmodel=None):
''' Parameter model is your Python model's metadata and conn is an active database connection. ''' """
:param model: Python model's metadata
:param conn: active database connection.
"""
self.model = model self.model = model
self.conn = conn self.conn = conn
if not excludeTables: excludeTables = [] # [] can't be default value in Python parameter if not excludeTables:
# [] can't be default value in Python parameter
excludeTables = []
self.excludeTables = excludeTables self.excludeTables = excludeTables
if oldmodel: if oldmodel:
self.reflected_model = oldmodel self.reflected_model = oldmodel
else: else:
self.reflected_model = sqlalchemy.MetaData(conn, reflect=True) self.reflected_model = sqlalchemy.MetaData(conn, reflect=True)
self.tablesMissingInDatabase, self.tablesMissingInModel, self.tablesWithDiff = [], [], [] self.tablesMissingInDatabase, self.tablesMissingInModel, \
self.tablesWithDiff = [], [], []
self.colDiffs = {} self.colDiffs = {}
self.compareModelToDatabase() self.compareModelToDatabase()
def compareModelToDatabase(self): def compareModelToDatabase(self):
''' Do actual comparison. ''' """
Do actual comparison.
"""
# Setup common variables. # Setup common variables.
cc = self.conn.contextual_connect() cc = self.conn.contextual_connect()
schemagenerator = self.conn.dialect.schemagenerator(self.conn.dialect, cc) schemagenerator = self.conn.dialect.schemagenerator(
self.conn.dialect, cc)
# For each in model, find missing in database. # For each in model, find missing in database.
for modelName, modelTable in self.model.tables.items(): for modelName, modelTable in self.model.tables.items():
if modelName in self.excludeTables: if modelName in self.excludeTables:
@ -49,95 +68,130 @@ class SchemaDiff(object):
pass pass
else: else:
self.tablesMissingInDatabase.append(modelTable) self.tablesMissingInDatabase.append(modelTable)
# For each in database, find missing in model. # For each in database, find missing in model.
for reflectedName, reflectedTable in self.reflected_model.tables.items(): for reflectedName, reflectedTable in \
self.reflected_model.tables.items():
if reflectedName in self.excludeTables: if reflectedName in self.excludeTables:
continue continue
modelTable = self.model.tables.get(reflectedName, None) modelTable = self.model.tables.get(reflectedName, None)
if modelTable: if modelTable:
# Table exists. # Table exists.
# Find missing columns in database. # Find missing columns in database.
for modelCol in modelTable.columns: for modelCol in modelTable.columns:
databaseCol = reflectedTable.columns.get(modelCol.name, None) databaseCol = reflectedTable.columns.get(modelCol.name,
None)
if databaseCol: if databaseCol:
pass pass
else: else:
self.storeColumnMissingInDatabase(modelTable, modelCol) self.storeColumnMissingInDatabase(modelTable, modelCol)
# Find missing columns in model. # Find missing columns in model.
for databaseCol in reflectedTable.columns: for databaseCol in reflectedTable.columns:
modelCol = modelTable.columns.get(databaseCol.name, None) modelCol = modelTable.columns.get(databaseCol.name, None)
if modelCol: if modelCol:
# Compare attributes of column. # Compare attributes of column.
modelDecl = schemagenerator.get_column_specification(modelCol) modelDecl = \
databaseDecl = schemagenerator.get_column_specification(databaseCol) schemagenerator.get_column_specification(
modelCol)
databaseDecl = \
schemagenerator.get_column_specification(
databaseCol)
if modelDecl != databaseDecl: if modelDecl != databaseDecl:
# Unfortunately, sometimes the database decl won't quite match the model, even though they're the same. # Unfortunately, sometimes the database
mc, dc = modelCol.type.__class__, databaseCol.type.__class__ # decl won't quite match the model, even
if (issubclass(mc, dc) or issubclass(dc, mc)) and modelCol.nullable == databaseCol.nullable: # though they're the same.
mc, dc = modelCol.type.__class__, \
databaseCol.type.__class__
if (issubclass(mc, dc) \
or issubclass(dc, mc)) \
and modelCol.nullable == \
databaseCol.nullable:
# Types and nullable are the same. # Types and nullable are the same.
pass pass
else: else:
self.storeColumnDiff(modelTable, modelCol, databaseCol, modelDecl, databaseDecl) self.storeColumnDiff(
modelTable, modelCol, databaseCol,
modelDecl, databaseDecl)
else: else:
self.storeColumnMissingInModel(modelTable, databaseCol) self.storeColumnMissingInModel(modelTable, databaseCol)
else: else:
self.tablesMissingInModel.append(reflectedTable) self.tablesMissingInModel.append(reflectedTable)
def __str__(self): def __str__(self):
''' Summarize differences. ''' ''' Summarize differences. '''
def colDiffDetails(): def colDiffDetails():
colout = [] colout = []
for table in self.tablesWithDiff: for table in self.tablesWithDiff:
tableName = table.name tableName = table.name
missingInDatabase, missingInModel, diffDecl = self.colDiffs[tableName] missingInDatabase, missingInModel, diffDecl = \
self.colDiffs[tableName]
if missingInDatabase: if missingInDatabase:
colout.append(' %s missing columns in database: %s' % (tableName, ', '.join([col.name for col in missingInDatabase]))) colout.append(
' %s missing columns in database: %s' % \
(tableName, ', '.join(
[col.name for col in missingInDatabase])))
if missingInModel: if missingInModel:
colout.append(' %s missing columns in model: %s' % (tableName, ', '.join([col.name for col in missingInModel]))) colout.append(
' %s missing columns in model: %s' % \
(tableName, ', '.join(
[col.name for col in missingInModel])))
if diffDecl: if diffDecl:
colout.append(' %s with different declaration of columns in database: %s' % (tableName, str(diffDecl))) colout.append(
' %s with different declaration of columns\
in database: %s' % (tableName, str(diffDecl)))
return colout return colout
out = [] out = []
if self.tablesMissingInDatabase: if self.tablesMissingInDatabase:
out.append(' tables missing in database: %s' % ', '.join([table.name for table in self.tablesMissingInDatabase])) out.append(
' tables missing in database: %s' % \
', '.join(
[table.name for table in self.tablesMissingInDatabase]))
if self.tablesMissingInModel: if self.tablesMissingInModel:
out.append(' tables missing in model: %s' % ', '.join([table.name for table in self.tablesMissingInModel])) out.append(
' tables missing in model: %s' % \
', '.join(
[table.name for table in self.tablesMissingInModel]))
if self.tablesWithDiff: if self.tablesWithDiff:
out.append(' tables with differences: %s' % ', '.join([table.name for table in self.tablesWithDiff])) out.append(
' tables with differences: %s' % \
', '.join([table.name for table in self.tablesWithDiff]))
if out: if out:
out.insert(0, 'Schema diffs:') out.insert(0, 'Schema diffs:')
out.extend(colDiffDetails()) out.extend(colDiffDetails())
return '\n'.join(out) return '\n'.join(out)
else: else:
return 'No schema diffs' return 'No schema diffs'
#__repr__ = __str__
def __len__(self): def __len__(self):
''' Used in bool evaluation, return of 0 means no diffs. ''' """
return len(self.tablesMissingInDatabase) + len(self.tablesMissingInModel) + len(self.tablesWithDiff) Used in bool evaluation, return of 0 means no diffs.
"""
return len(self.tablesMissingInDatabase) + \
len(self.tablesMissingInModel) + len(self.tablesWithDiff)
def storeColumnMissingInDatabase(self, table, col): def storeColumnMissingInDatabase(self, table, col):
if table not in self.tablesWithDiff: if table not in self.tablesWithDiff:
self.tablesWithDiff.append(table) self.tablesWithDiff.append(table)
missingInDatabase, missingInModel, diffDecl = self.colDiffs.setdefault(table.name, ([], [], [])) missingInDatabase, missingInModel, diffDecl = \
self.colDiffs.setdefault(table.name, ([], [], []))
missingInDatabase.append(col) missingInDatabase.append(col)
def storeColumnMissingInModel(self, table, col): def storeColumnMissingInModel(self, table, col):
if table not in self.tablesWithDiff: if table not in self.tablesWithDiff:
self.tablesWithDiff.append(table) self.tablesWithDiff.append(table)
missingInDatabase, missingInModel, diffDecl = self.colDiffs.setdefault(table.name, ([], [], [])) missingInDatabase, missingInModel, diffDecl = \
self.colDiffs.setdefault(table.name, ([], [], []))
missingInModel.append(col) missingInModel.append(col)
def storeColumnDiff(self, table, modelCol, databaseCol, modelDecl, databaseDecl): def storeColumnDiff(self, table, modelCol, databaseCol, modelDecl,
databaseDecl):
if table not in self.tablesWithDiff: if table not in self.tablesWithDiff:
self.tablesWithDiff.append(table) self.tablesWithDiff.append(table)
missingInDatabase, missingInModel, diffDecl = self.colDiffs.setdefault(table.name, ([], [], [])) missingInDatabase, missingInModel, diffDecl = \
diffDecl.append( (modelCol, databaseCol, modelDecl, databaseDecl) ) self.colDiffs.setdefault(table.name, ([], [], []))
diffDecl.append((modelCol, databaseCol, modelDecl, databaseDecl))