Support postgresql in dblayer

Change-Id: I7131273d137c0d42991a0cbd17dd9bc2dbf04d5c
Closes-bug: #1545659

.config

@@ -1,7 +1,7 @@
-
 celery_broker: sqla+sqlite:////tmp/celery.db
 celery_backend: db+sqlite:////tmp/celery.db
 solar_db: sqlite:////tmp/solar.db
+# solar_db: postgresql:///postgres:password@10.0.0.2:5432/solardb
 # solar_db: riak://10.0.0.2:8087
 riak_ensemble: False
 lock_bucket_type: ''

bootstrap/playbooks/tasks/base.yaml

@@ -38,6 +38,9 @@
   # for tests on jenkins
   - sshpass
 
+  # for pg backend
+  - python-psycopg2
+
 - name: Uninstall packages
   apt: name={{ item }} state=absent
   with_items:

requirements.txt

@@ -22,10 +22,14 @@ pbr
 pydot
 bunch
 wrapt
+
 # if you want to use riak backend then
 # riak
 # if you want to use sql backend then
 peewee
+# if you want to use sql + postgresql then
+# psycopg2
+
 # if you want to use lua computable inputs
 # lupa
 
@@ -38,4 +42,3 @@ stevedore
 #zerorpc doesnt consume messages with >13.0.2, need to debug
 pyzmq==13.0.2
 zerorpc>=0.5.2
-

solar/dblayer/__init__.py

@@ -40,6 +40,7 @@ if _connection.mode == 'sqlite':
                 'pragmas': (('journal_mode', 'WAL'),
                             ('synchronous', 'NORMAL'))}
     opts.update(_connection_details.toDict())
+    opts.setdefault('db_class', 'SqliteDatabase')
     client = SqlClient(
         _connection.database,
         **opts)
@@ -60,6 +61,24 @@ elif _connection.mode == 'riak':
                             **opts)
     else:
         raise Exception('Unknown riak protocol %s', proto)
+
+elif _connection.mode == 'postgresql':
+    # TODO: collation has to be `C`
+    from solar.dblayer.sql_client import SqlClient
+    opts = {'autocommit': False}
+    opts.update(_connection_details.toDict())
+    if _connection.port:
+        _connection.port = int(_connection.port)
+    else:
+        _connection.port = None
+    opts["user"] = _connection.username
+    opts["host"] = _connection.host
+    opts["port"] = _connection.port
+    opts["password"] = _connection.password
+    # TODO: allow set Postgresql classes from playhouse
+    opts.setdefault('db_class', 'PostgresqlDatabase')
+    client = SqlClient(_connection.database,
+                       **opts)
 else:
     raise Exception('Unknown dblayer backend %s', C.solar_db)
 

solar/dblayer/locking.py

@@ -230,7 +230,7 @@ class RiakLock(_CRDTishLock):
     pass
 
 
-class SQLiteLock(_CRDTishLock):
+class SQLLock(_CRDTishLock):
 
     @classmethod
     def _end_start_session(cls, uid, identity):
@@ -267,12 +267,14 @@ class RiakEnsembleLock(_Lock):
             raise
 
 
-if _connection.mode == 'sqlite':
+if _connection.type == 'sql':
-    Lock = SQLiteLock
+    Lock = SQLLock
 elif _connection.mode == 'riak':
     if C.riak_ensemble:
         Lock = RiakEnsembleLock
     else:
         Lock = RiakLock
+else:
+    raise RuntimeError("Unsupported database connection setting")
 
 Waiter = SemaWaiter

solar/dblayer/solar_models.py

@@ -1072,7 +1072,7 @@ system log
 """
 
 _connection, _connection_details = parse_database_conn(C.solar_db)
-if _connection.mode == 'sqlite':
+if _connection.type == 'sql':
     class NegativeCounter(Model):
 
         count = Field(int, default=int)

solar/dblayer/sql_client.py

@@ -268,7 +268,10 @@ class Bucket(object):
 
         self._sql_model = type(table_name, (_SqlBucket, ), {'Meta': ModelMeta,
                                                             'bucket': self})
-        _idx_key = ForeignKeyField(self._sql_model, null=False, index=True)
+        _idx_key = ForeignKeyField(self._sql_model,
+                                   null=False,
+                                   index=True,
+                                   on_delete='cascade')
 
         class IdxMeta(object):
             db_table = idx_table_name
@@ -405,9 +408,9 @@ class SqlClient(object):
     search_dir = None
 
     def __init__(self, *args, **kwargs):
-        db_class_str = kwargs.pop("db_class", 'SqliteDatabase')
+        db_class_str = kwargs.pop("db_class")
         try:
-            mod, fromlist = db_class_str.split('.')
+            mod, fromlist = db_class_str.rsplit('.', 1)
         except ValueError:
             mod = 'peewee'
             fromlist = db_class_str

solar/utils.py

@@ -167,6 +167,7 @@ def parse_database_conn(name):
     m = regex.match(name)
     if m is not None:
         groups = m.groupdict()
+        groups['type'] = 'riak' if groups['mode'] == 'riak' else 'sql'
         return Bunch(groups), Bunch(opts)
     else:
         raise Exception("Invalid database connection string: %r "