Merge "Externalize database connection pool parameters"

docs/source/_static/shipyard.conf.sample

@@ -47,7 +47,25 @@
 # The database for airflow (string value)
 #postgresql_airflow_db = postgresql+psycopg2://shipyard:changeme@postgresql.ucp:5432/airflow
 
-# The direcotry containing the alembic.ini file (string value)
+# The SQLalchemy database connection pool size. (integer value)
+#pool_size = 15
+
+# Should DB connections be validated prior to use. (boolean value)
+#pool_pre_ping = true
+
+# How long a request for a connection should wait before one becomes available.
+# (integer value)
+#pool_timeout = 30
+
+# How many connections above pool_size are allowed to be open during high
+# usage. (integer value)
+#pool_overflow = 10
+
+# Time, in seconds, when a connection should be closed and re-established. -1
+# for no recycling. (integer value)
+#connection_recycle = -1
+
+# The directory containing the alembic.ini file (string value)
 #alembic_ini_path = /home/shipyard/shipyard
 
 

images/airflow/requirements.txt

@@ -26,3 +26,4 @@ kubernetes>=6.0.0
 git+https://git.openstack.org/openstack/airship-deckhand@177675e96fffcda9799c68bbce831424c1167020#egg=deckhand
 git+https://github.com/att-comdev/drydock.git@8af92eaf29ca0dd6a129748c132ea7f6593eae83#egg=drydock_provisioner
 git+https://github.com/att-comdev/armada.git@7a2ba22ab12a3f1f180b6af4085972ba44853377#egg=armada
+

src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample

@@ -47,7 +47,25 @@
 # The database for airflow (string value)
 #postgresql_airflow_db = postgresql+psycopg2://shipyard:changeme@postgresql.ucp:5432/airflow
 
-# The direcotry containing the alembic.ini file (string value)
+# The SQLalchemy database connection pool size. (integer value)
+#pool_size = 15
+
+# Should DB connections be validated prior to use. (boolean value)
+#pool_pre_ping = true
+
+# How long a request for a connection should wait before one becomes available.
+# (integer value)
+#pool_timeout = 30
+
+# How many connections above pool_size are allowed to be open during high
+# usage. (integer value)
+#pool_overflow = 10
+
+# Time, in seconds, when a connection should be closed and re-established. -1
+# for no recycling. (integer value)
+#connection_recycle = -1
+
+# The directory containing the alembic.ini file (string value)
 #alembic_ini_path = /home/shipyard/shipyard
 
 

src/bin/shipyard_airflow/requirements.txt

@@ -30,7 +30,7 @@ python-dateutil==2.6.1
 python-memcached==1.58
 requests==2.18.4
 setuptools==39.0.1
-SQLAlchemy==1.1.13
+SQLAlchemy==1.2.8
 ulid==1.1
 uwsgi==2.0.15
 

src/bin/shipyard_airflow/shipyard_airflow/conf/config.py

@@ -58,10 +58,38 @@ SECTIONS = [
                 ),
                 help='The database for airflow'
             ),
+            cfg.IntOpt(
+                'pool_size',
+                default=15,
+                help='The SQLalchemy database connection pool size.'
+            ),
+            cfg.BoolOpt(
+                'pool_pre_ping',
+                default=True,
+                help='Should DB connections be validated prior to use.'
+            ),
+            cfg.IntOpt(
+                'pool_timeout',
+                default=30,
+                help=('How long a request for a connection should wait before '
+                      'one becomes available.')
+            ),
+            cfg.IntOpt(
+                'pool_overflow',
+                default=10,
+                help=('How many connections above pool_size are allowed to be '
+                      'open during high usage.')
+            ),
+            cfg.IntOpt(
+                'connection_recycle',
+                default=-1,
+                help=('Time, in seconds, when a connection should be closed '
+                      'and re-established. -1 for no recycling.')
+            ),
             cfg.StrOpt(
                 'alembic_ini_path',
                 default='/home/shipyard/shipyard',
-                help='The direcotry containing the alembic.ini file'
+                help='The directory containing the alembic.ini file'
             ),
         ]
     ),

src/bin/shipyard_airflow/shipyard_airflow/db/common_db.py

@@ -17,10 +17,12 @@ module for reused or baseclass portions of DB access
 
 import logging
 
+from oslo_config import cfg
 import sqlalchemy
 
 from shipyard_airflow.errors import DatabaseError
 
+CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
 
@@ -31,6 +33,14 @@ class DbAccess:
 
     def __init__(self):
         self.engine = None
+        # Descendent classes can override these after construciton, but before
+        # invoking get_engine, if there is a need to tailor the values
+        # for that specific database.
+        self.pool_size = CONF.base.pool_size
+        self.max_overflow = CONF.base.pool_overflow
+        self.pool_pre_ping = CONF.base.pool_pre_ping
+        self.pool_recycle = CONF.base.connection_recycle
+        self.pool_timeout = CONF.base.pool_timeout
 
     def get_connection_string(self):
         """
@@ -53,7 +63,19 @@ class DbAccess:
         try:
             connection_string = self.get_connection_string()
             if connection_string is not None and self.engine is None:
-                self.engine = sqlalchemy.create_engine(connection_string)
+                LOG.info('Initializing connection <%s> with pool size: %d, '
+                         'max overflow: %d, pool pre ping: %s, pool '
+                         'recycle: %d, and pool timeout: %d',
+                         connection_string, self.pool_size, self.max_overflow,
+                         self.pool_pre_ping, self.pool_recycle,
+                         self.pool_timeout)
+                self.engine = sqlalchemy.create_engine(
+                    connection_string, pool_size=self.pool_size,
+                    max_overflow=self.max_overflow,
+                    pool_pre_ping=self.pool_pre_ping,
+                    pool_recycle=self.pool_recycle,
+                    pool_timeout=self.pool_timeout
+                )
             if self.engine is None:
                 self._raise_invalid_db_config(
                     connection_string=connection_string