diff --git a/.config b/.config index 00eccfe9..f22f8b1b 100644 --- a/.config +++ b/.config @@ -1,7 +1,7 @@ - celery_broker: sqla+sqlite:////tmp/celery.db celery_backend: db+sqlite:////tmp/celery.db solar_db: sqlite:////tmp/solar.db +# solar_db: postgresql:///postgres:password@10.0.0.2:5432/solardb # solar_db: riak://10.0.0.2:8087 riak_ensemble: False lock_bucket_type: '' diff --git a/bootstrap/playbooks/tasks/base.yaml b/bootstrap/playbooks/tasks/base.yaml index 27947016..047a4a37 100644 --- a/bootstrap/playbooks/tasks/base.yaml +++ b/bootstrap/playbooks/tasks/base.yaml @@ -38,6 +38,9 @@ # for tests on jenkins - sshpass + # for pg backend + - python-psycopg2 + - name: Uninstall packages apt: name={{ item }} state=absent with_items: diff --git a/requirements.txt b/requirements.txt index 438763ca..5788e71b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,10 +22,14 @@ pbr pydot bunch wrapt + # if you want to use riak backend then # riak # if you want to use sql backend then peewee +# if you want to use sql + postgresql then +# psycopg2 + # if you want to use lua computable inputs # lupa @@ -38,4 +42,3 @@ stevedore #zerorpc doesnt consume messages with >13.0.2, need to debug pyzmq==13.0.2 zerorpc>=0.5.2 - diff --git a/solar/dblayer/__init__.py b/solar/dblayer/__init__.py index ed3e383c..b6577799 100644 --- a/solar/dblayer/__init__.py +++ b/solar/dblayer/__init__.py @@ -40,6 +40,7 @@ if _connection.mode == 'sqlite': 'pragmas': (('journal_mode', 'WAL'), ('synchronous', 'NORMAL'))} opts.update(_connection_details.toDict()) + opts.setdefault('db_class', 'SqliteDatabase') client = SqlClient( _connection.database, **opts) @@ -60,6 +61,24 @@ elif _connection.mode == 'riak': **opts) else: raise Exception('Unknown riak protocol %s', proto) + +elif _connection.mode == 'postgresql': + # TODO: collation has to be `C` + from solar.dblayer.sql_client import SqlClient + opts = {'autocommit': False} + opts.update(_connection_details.toDict()) + if _connection.port: + _connection.port = int(_connection.port) + else: + _connection.port = None + opts["user"] = _connection.username + opts["host"] = _connection.host + opts["port"] = _connection.port + opts["password"] = _connection.password + # TODO: allow set Postgresql classes from playhouse + opts.setdefault('db_class', 'PostgresqlDatabase') + client = SqlClient(_connection.database, + **opts) else: raise Exception('Unknown dblayer backend %s', C.solar_db) diff --git a/solar/dblayer/locking.py b/solar/dblayer/locking.py index ae81ee3d..f293f0af 100644 --- a/solar/dblayer/locking.py +++ b/solar/dblayer/locking.py @@ -230,7 +230,7 @@ class RiakLock(_CRDTishLock): pass -class SQLiteLock(_CRDTishLock): +class SQLLock(_CRDTishLock): @classmethod def _end_start_session(cls, uid, identity): @@ -267,12 +267,14 @@ class RiakEnsembleLock(_Lock): raise -if _connection.mode == 'sqlite': - Lock = SQLiteLock +if _connection.type == 'sql': + Lock = SQLLock elif _connection.mode == 'riak': if C.riak_ensemble: Lock = RiakEnsembleLock else: Lock = RiakLock +else: + raise RuntimeError("Unsupported database connection setting") Waiter = SemaWaiter diff --git a/solar/dblayer/solar_models.py b/solar/dblayer/solar_models.py index f5c092cb..80a0578e 100644 --- a/solar/dblayer/solar_models.py +++ b/solar/dblayer/solar_models.py @@ -1072,7 +1072,7 @@ system log """ _connection, _connection_details = parse_database_conn(C.solar_db) -if _connection.mode == 'sqlite': +if _connection.type == 'sql': class NegativeCounter(Model): count = Field(int, default=int) diff --git a/solar/dblayer/sql_client.py b/solar/dblayer/sql_client.py index 94dd0acb..7213724a 100644 --- a/solar/dblayer/sql_client.py +++ b/solar/dblayer/sql_client.py @@ -268,7 +268,10 @@ class Bucket(object): self._sql_model = type(table_name, (_SqlBucket, ), {'Meta': ModelMeta, 'bucket': self}) - _idx_key = ForeignKeyField(self._sql_model, null=False, index=True) + _idx_key = ForeignKeyField(self._sql_model, + null=False, + index=True, + on_delete='cascade') class IdxMeta(object): db_table = idx_table_name @@ -405,9 +408,9 @@ class SqlClient(object): search_dir = None def __init__(self, *args, **kwargs): - db_class_str = kwargs.pop("db_class", 'SqliteDatabase') + db_class_str = kwargs.pop("db_class") try: - mod, fromlist = db_class_str.split('.') + mod, fromlist = db_class_str.rsplit('.', 1) except ValueError: mod = 'peewee' fromlist = db_class_str diff --git a/solar/utils.py b/solar/utils.py index 50c4ea38..fa7708a0 100644 --- a/solar/utils.py +++ b/solar/utils.py @@ -167,6 +167,7 @@ def parse_database_conn(name): m = regex.match(name) if m is not None: groups = m.groupdict() + groups['type'] = 'riak' if groups['mode'] == 'riak' else 'sql' return Bunch(groups), Bunch(opts) else: raise Exception("Invalid database connection string: %r "