Revive Alembic-based database migration logic.

Alembic migrations were introduced to Barbican a while ago but have not been
used recently. This CR revisits the migration logic and ensures that time-zero
database tables creation is distinguished from Alembic-base schema updates.
The associated database migration script is also revived, allowing for
schema migrations outside of the Barbican boot process.

Change-Id: I3e7fd7ac3f629da18329c22ad11cb9ccc1b7f9f9
Implements: blueprint db-revive-migrations

.gitignore

@@ -24,6 +24,8 @@ pip-log.txt
 *.err.log
 
 # Unit test / coverage reports
+cover
+.testrepository
 .coverage
 .tox
 nosetests.xml

barbican/model/migration/alembic.ini

@@ -12,7 +12,7 @@ script_location = %(here)s/alembic_migrations
 # revision_environment = false
 
 # default to an empty string because the Barbican migration process will
-# extract the correct value and set it programatically before alemic is fully
+# extract the correct value and set it programatically before alembic is fully
 # invoked.
 sqlalchemy.url =
 #sqlalchemy.url = driver://user:pass@localhost/dbname

barbican/model/migration/commands.py

@@ -56,6 +56,13 @@ def downgrade(to_version, sql_url=None):
         alembic_command.downgrade(alembic_cfg, to_version)
 
 
+def stamp(to_version='head', sql_url=None):
+    """Stamp the specified version, with no migration performed."""
+    alembic_cfg = init_config(sql_url)
+    if alembic_cfg:
+        alembic_command.stamp(alembic_cfg, to_version)
+
+
 def generate(autogenerate=True, message='generate changes', sql_url=None):
     """Generate a version file."""
     alembic_cfg = init_config(sql_url)

barbican/model/repositories.py

@@ -141,11 +141,21 @@ def get_engine():
             sa_logger.setLevel(logging.DEBUG)
 
         if CONF.db_auto_create:
-            LOG.info(_('auto-creating barbican registry DB'))
-            models.register_models(_ENGINE)
+            meta = sqlalchemy.MetaData()
+            meta.reflect(bind=_ENGINE)
+            tables = meta.tables
+            if tables and 'alembic_version' in tables:
+                # Upgrade the database to the latest version.
+                LOG.info(_('Updating schema to latest version'))
+                commands.upgrade()
+            else:
+                # Create database tables from our models.
+                LOG.info(_('Auto-creating barbican registry DB'))
+                models.register_models(_ENGINE)
+
+                # Sync the alembic version 'head' with current models.
+                commands.stamp()
 
-            # Upgrade the database to the latest version.
-            commands.upgrade()
         else:
             LOG.info(_('not auto-creating barbican registry DB'))
 

bin/barbican-db-manage.py

@@ -8,6 +8,8 @@ import argparse
 sys.path.insert(0, os.getcwd())
 
 from barbican.model.migration import commands
+from barbican.openstack.common import log
+
 
 class DatabaseManager:
     """
@@ -28,7 +30,7 @@ class DatabaseManager:
         """Create top-level parser and arguments."""
         parser = argparse.ArgumentParser(description='Barbican DB manager.')
         parser.add_argument('--dburl', '-d', default=None,
-                             help='URL to the database)')
+                             help='URL to the database.')
 
         return parser
 
@@ -90,6 +92,11 @@ class DatabaseManager:
 
 
 def main():
+    # Import and configure logging.
+    log.setup('barbican-db-manage')
+    LOG = log.getLogger(__name__)
+    LOG.debug("Performing database schema migration...")
+
     dm = DatabaseManager()
     dm.execute()
 

requirements.txt

@@ -14,7 +14,7 @@ pbr>=0.5.21,<1.0
 pycrypto>=2.6
 pysqlite
 python-keystoneclient>=0.4.1
-SQLAlchemy>=0.7.8,<=0.7.99
+SQLAlchemy>=0.7.8,<=0.9.99
 stevedore>=0.12
 WebOb>=1.2.3,<1.3
 wsgiref>=0.1.2

rpmbuild/SPECS/barbican.spec

@@ -43,6 +43,7 @@ install -m 644 etc/barbican/policy.json $RPM_BUILD_ROOT/etc/barbican
 install -m 644 etc/init/barbican-api.conf $RPM_BUILD_ROOT/etc/init
 install -m 644 etc/init/barbican-worker.conf $RPM_BUILD_ROOT/etc/init
 install bin/barbican-worker.py $RPM_BUILD_ROOT/usr/bin
+install bin/barbican-db-manage.py $RPM_BUILD_ROOT/usr/bin
 install -m 644 -D etc/barbican/barbican* $RPM_BUILD_ROOT/etc/barbican
 install -m 644 -D etc/barbican/vassals/*.ini $RPM_BUILD_ROOT/etc/barbican/vassals
 touch $RPM_BUILD_ROOT/var/log/barbican/barbican-api.log
@@ -83,6 +84,7 @@ Barbican Key Manager API daemon
 %verify(not md5 size mtime) %attr(0750, barbican,root) /var/log/barbican/barbican-api.log
 /etc/logrotate.d/barbican-api
 %attr(0755,root,root) /usr/bin/barbican.sh
+%attr(0755,root,root) /usr/bin/barbican-db-manage.py
 %config(noreplace) /etc/init/barbican-api.conf
 %config(noreplace) /etc/barbican/*
 
@@ -109,6 +111,7 @@ Barbican Key Manager worker daemon
 %verify(not md5 size mtime) %attr(0750, barbican,root) /var/log/barbican/barbican-api.log
 /etc/logrotate.d/barbican-api
 %attr(0755,root,root) /usr/bin/barbican-worker.py
+%attr(0755,root,root) /usr/bin/barbican-db-manage.py
 %config(noreplace) /etc/init/barbican-worker.conf
 %config(noreplace) /etc/barbican/*
 

tox.ini

@@ -20,4 +20,5 @@ commands = {toxinidir}/tools/hacking.sh {posargs}
 # E711 ignored because of sqlalchemy override of == None
 # H ignored because it's not H clean
 ignore = E711,H
-exclude = .git,.idea,.tox,bin,dist,debian,rpmbuild,tools,*.egg-info,*openstack/common,contrib,functionaltests
+exclude = .git,.idea,.tox,bin,dist,debian,rpmbuild,tools,*.egg-info,*openstack/common,contrib,
+          functionaltests,*alembic_migrations/versions