Refactored verifier config into a separate module

stacktach/views.py

@@ -288,6 +288,7 @@ def _process_exists(raw, notification):
 def _process_glance_usage(raw, notification):
     notification.save_usage(raw)
 
+
 def _process_glance_delete(raw, notification):
     notification.save_delete(raw)
 

tests/unit/test_verifier_db.py

@@ -46,11 +46,16 @@ from utils import OS_VERSION_2
 from utils import TENANT_ID_1
 from utils import TENANT_ID_2
 from utils import INSTANCE_TYPE_ID_1
+from utils import TICK_TIME
+from utils import SETTLE_TIME
+from utils import SETTLE_UNITS
 from verifier import dbverifier
+from verifier import config as verifier_config
 from verifier import AmbiguousResults
 from verifier import FieldMismatch
 from verifier import NotFound
 from verifier import VerificationException
+import stubout
 
 
 class VerifierTestCase(unittest.TestCase):
@@ -83,40 +88,28 @@ class VerifierTestCase(unittest.TestCase):
         models.JsonReport.objects = self.mox.CreateMockAnything()
         self._setup_verifier()
 
+        self.stubs = stubout.StubOutForTesting()
+        self.stubs.Set(verifier_config, 'durable_queue', lambda: True)
+        self.stubs.Set(verifier_config, 'source_topics', lambda:
+            {'nova': ['notifications.info']})
+        self.stubs.Set(verifier_config, 'enable_notifications', lambda: True)
+        self.stubs.Set(verifier_config, 'host', lambda: '10.0.0.1')
+        self.stubs.Set(verifier_config, 'virtual_host', lambda: '/')
+        self.stubs.Set(verifier_config, 'port', lambda: '5672')
+        self.stubs.Set(verifier_config, 'userid', lambda: 'rabbit')
+        self.stubs.Set(verifier_config, 'password', lambda: 'password')
+        self.stubs.Set(verifier_config, 'tick_time', lambda: TICK_TIME)
+        self.stubs.Set(verifier_config, 'settle_time', lambda: SETTLE_TIME)
+        self.stubs.Set(verifier_config, 'settle_units', lambda: SETTLE_UNITS)
+
     def _setup_verifier(self):
-        self.config = {
-            "tick_time": 30,
-            "settle_time": 5,
-            "settle_units": "minutes",
-            "pool_size": 2,
-            "enable_notifications": False,
-        }
         self.pool = self.mox.CreateMockAnything()
         self.reconciler = self.mox.CreateMockAnything()
-        self.verifier = dbverifier.Verifier(self.config,
+        self.verifier = dbverifier.Verifier('nova', pool=self.pool,
-                                            pool=self.pool,
                                             rec=self.reconciler)
-
-        self.config_notif = {
-            "tick_time": 30,
-            "settle_time": 5,
-            "settle_units": "minutes",
-            "pool_size": 2,
-            "enable_notifications": True,
-            "rabbit": {
-                "durable_queue": False,
-                "host": "10.0.0.1",
-                "port": 5672,
-                "userid": "rabbit",
-                "password": "rabbit",
-                "virtual_host": "/",
-                "exchange_name": "stacktach",
-            }
-        }
         self.pool_notif = self.mox.CreateMockAnything()
         self.reconciler_notif = self.mox.CreateMockAnything()
-        self.verifier_notif = dbverifier.Verifier(self.config_notif,
+        self.verifier_notif = dbverifier.Verifier('nova', pool=self.pool_notif,
-                                                  pool=self.pool_notif,
                                                   rec=self.reconciler)
 
     def tearDown(self):
@@ -125,6 +118,7 @@ class VerifierTestCase(unittest.TestCase):
         self.pool = None
         self.verifier_notif = None
         self.pool_notif = None
+        self.stubs.UnsetAll()
 
     def test_verify_for_launch(self):
         exist = self.mox.CreateMockAnything()
@@ -1024,13 +1018,14 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.VerifyAll()
 
     def test_run_notifications(self):
+        self.stubs.Set(verifier_config, 'durable_queue', lambda: False)
         self.mox.StubOutWithMock(dbverifier, '_create_exchange')
         exchange = self.mox.CreateMockAnything()
-        dbverifier._create_exchange('stacktach', 'topic', durable=False)\
+        dbverifier._create_exchange('nova', 'topic', durable=False)\
                   .AndReturn(exchange)
         self.mox.StubOutWithMock(dbverifier, '_create_connection')
         conn = self.mox.CreateMockAnything()
-        dbverifier._create_connection(self.config_notif).AndReturn(conn)
+        dbverifier._create_connection().AndReturn(conn)
         conn.__enter__().AndReturn(conn)
         self.mox.StubOutWithMock(self.verifier_notif, '_run')
         self.verifier_notif._run(callback=mox.Not(mox.Is(None)))
@@ -1040,13 +1035,14 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.VerifyAll()
 
     def test_run_notifications_with_routing_keys(self):
+        self.stubs.Set(verifier_config, 'durable_queue', lambda: False)
         self.mox.StubOutWithMock(dbverifier, '_create_exchange')
         exchange = self.mox.CreateMockAnything()
-        dbverifier._create_exchange('stacktach', 'topic', durable=False) \
+        dbverifier._create_exchange('nova', 'topic', durable=False) \
             .AndReturn(exchange)
         self.mox.StubOutWithMock(dbverifier, '_create_connection')
         conn = self.mox.CreateMockAnything()
-        dbverifier._create_connection(self.config_notif).AndReturn(conn)
+        dbverifier._create_connection().AndReturn(conn)
         conn.__enter__().AndReturn(conn)
         self.mox.StubOutWithMock(self.verifier_notif, '_run')
         self.verifier_notif._run(callback=mox.Not(mox.Is(None)))
@@ -1056,6 +1052,7 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.VerifyAll()
 
     def test_run_no_notifications(self):
+        self.stubs.Set(verifier_config, 'enable_notifications', lambda: False)
         self.mox.StubOutWithMock(self.verifier, '_run')
         self.verifier._run()
         self.mox.ReplayAll()
@@ -1063,13 +1060,14 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.VerifyAll()
 
     def test_run_once_notifications(self):
+        self.stubs.Set(verifier_config, 'durable_queue', lambda: False)
         self.mox.StubOutWithMock(dbverifier, '_create_exchange')
         exchange = self.mox.CreateMockAnything()
-        dbverifier._create_exchange('stacktach', 'topic', durable=False) \
+        dbverifier._create_exchange('nova', 'topic', durable=False) \
             .AndReturn(exchange)
         self.mox.StubOutWithMock(dbverifier, '_create_connection')
         conn = self.mox.CreateMockAnything()
-        dbverifier._create_connection(self.config_notif).AndReturn(conn)
+        dbverifier._create_connection().AndReturn(conn)
         conn.__enter__().AndReturn(conn)
         self.mox.StubOutWithMock(self.verifier_notif, '_run_once')
         self.verifier_notif._run_once(callback=mox.Not(mox.Is(None)))
@@ -1079,6 +1077,8 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.VerifyAll()
 
     def test_run_once_no_notifications(self):
+        self.mox.StubOutWithMock(verifier_config, 'enable_notifications')
+        verifier_config.enable_notifications().AndReturn(False)
         self.mox.StubOutWithMock(self.verifier, '_run_once')
         self.verifier._run_once()
         self.mox.ReplayAll()
@@ -1086,6 +1086,7 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.VerifyAll()
 
     def test_run_full_no_notifications(self):
+        self.stubs.Set(verifier_config, 'enable_notifications', lambda: False)
         self.verifier.reconcile = True
         self.mox.StubOutWithMock(self.verifier, '_keep_running')
         self.verifier._keep_running().AndReturn(True)
@@ -1096,9 +1097,7 @@ class VerifierTestCase(unittest.TestCase):
         start = datetime.datetime.utcnow()
         self.mox.StubOutWithMock(self.verifier, '_utcnow')
         self.verifier._utcnow().AndReturn(start)
-        settle_time = self.config['settle_time']
+        settle_offset = {SETTLE_UNITS: SETTLE_TIME}
-        settle_units = self.config['settle_units']
-        settle_offset = {settle_units: settle_time}
         ending_max = start - datetime.timedelta(**settle_offset)
         self.mox.StubOutWithMock(self.verifier, 'verify_for_range')
         self.verifier.verify_for_range(ending_max, callback=None)
@@ -1115,7 +1114,7 @@ class VerifierTestCase(unittest.TestCase):
         self.verifier.reconcile_failed()
         fake_transaction.__exit__(None, None, None)
         self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
-        time.sleep(self.config['tick_time'])
+        time.sleep(TICK_TIME)
         self.verifier._keep_running().AndReturn(False)
         self.mox.ReplayAll()
         self.verifier.run()
@@ -1132,9 +1131,7 @@ class VerifierTestCase(unittest.TestCase):
         start = datetime.datetime.utcnow()
         self.mox.StubOutWithMock(self.verifier_notif, '_utcnow')
         self.verifier_notif._utcnow().AndReturn(start)
-        settle_time = self.config['settle_time']
+        settle_offset = {SETTLE_UNITS: SETTLE_TIME}
-        settle_units = self.config['settle_units']
-        settle_offset = {settle_units: settle_time}
         ending_max = start - datetime.timedelta(**settle_offset)
         self.mox.StubOutWithMock(self.verifier_notif, 'verify_for_range')
         self.verifier_notif.verify_for_range(ending_max,
@@ -1152,20 +1149,19 @@ class VerifierTestCase(unittest.TestCase):
         self.verifier_notif.reconcile_failed()
         fake_transaction.__exit__(None, None, None)
         self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
-        time.sleep(self.config['tick_time'])
+        time.sleep(TICK_TIME)
         self.verifier_notif._keep_running().AndReturn(False)
         self.mox.ReplayAll()
         self.verifier_notif.run()
         self.mox.VerifyAll()
 
     def test_run_once_full_no_notifications(self):
+        self.stubs.Set(verifier_config, 'enable_notifications', lambda: False)
         self.verifier.reconcile = True
         start = datetime.datetime.utcnow()
         self.mox.StubOutWithMock(self.verifier, '_utcnow')
         self.verifier._utcnow().AndReturn(start)
-        settle_time = self.config['settle_time']
+        settle_offset = {SETTLE_UNITS: SETTLE_TIME}
-        settle_units = self.config['settle_units']
-        settle_offset = {settle_units: settle_time}
         ending_max = start - datetime.timedelta(**settle_offset)
         self.mox.StubOutWithMock(self.verifier, 'verify_for_range')
         self.verifier.verify_for_range(ending_max, callback=None)
@@ -1181,7 +1177,7 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.StubOutWithMock(self.verifier, 'reconcile_failed')
         self.verifier.reconcile_failed()
         self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
-        time.sleep(self.config['tick_time'])
+        time.sleep(TICK_TIME)
         self.mox.ReplayAll()
         self.verifier.run_once()
         self.mox.VerifyAll()
@@ -1191,9 +1187,7 @@ class VerifierTestCase(unittest.TestCase):
         start = datetime.datetime.utcnow()
         self.mox.StubOutWithMock(self.verifier_notif, '_utcnow')
         self.verifier_notif._utcnow().AndReturn(start)
-        settle_time = self.config['settle_time']
+        settle_offset = {SETTLE_UNITS: SETTLE_TIME}
-        settle_units = self.config['settle_units']
-        settle_offset = {settle_units: settle_time}
         ending_max = start - datetime.timedelta(**settle_offset)
         self.mox.StubOutWithMock(self.verifier_notif, 'verify_for_range')
         self.verifier_notif.verify_for_range(ending_max,
@@ -1210,7 +1204,7 @@ class VerifierTestCase(unittest.TestCase):
         self.mox.StubOutWithMock(self.verifier_notif, 'reconcile_failed')
         self.verifier_notif.reconcile_failed()
         self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
-        time.sleep(self.config['tick_time'])
+        time.sleep(TICK_TIME)
         self.mox.ReplayAll()
         self.verifier_notif.run_once()
         self.mox.VerifyAll()

tests/unit/utils.py

@@ -56,6 +56,10 @@ OS_VERSION_1 = "1"
 OS_VERSION_2 = "2"
 
 TIMESTAMP_1 = "2013-06-20 17:31:57.939614"
+SETTLE_TIME = 5
+SETTLE_UNITS = "minutes"
+TICK_TIME = 10
+
 
 def decimal_utc(t = datetime.datetime.utcnow()):
     return dt.dt_to_decimal(t)

verifier/config.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2013 - Rackspace Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+import json
+import os
+
+config_filename = os.environ.get('STACKTACH_VERIFIER_CONFIG',
+                                 'stacktach_verifier_config.json')
+
+try:
+    from local_settings import *
+    config_filename = STACKTACH_VERIFIER_CONFIG
+except ImportError:
+    pass
+
+config = None
+with open(config_filename, "r") as f:
+    config = json.load(f)
+
+
+def enable_notifications():
+    return config['enable_notifications']
+
+
+def source_topics():
+    return config['rabbit']['source_topics']
+
+
+def tick_time():
+    return config['tick_time']
+
+
+def settle_units():
+    return config['settle_units']
+
+
+def settle_time():
+    return config['settle_time']
+
+
+def reconcile():
+    return config.get('reconcile', False)
+
+
+def reconciler_config():
+    return config.get(
+        'reconciler_config', '/etc/stacktach/reconciler_config.json')
+
+
+def source_topics():
+    return config['rabbit']['source_topics']
+
+
+def pool_size():
+    return config['pool_size']
+
+
+def durable_queue():
+    return config['rabbit']['durable_queue']
+
+
+def host():
+    return config['rabbit']['host']
+
+
+def port():
+    return config['rabbit']['port']
+
+
+def userid():
+    return config['rabbit']['userid']
+
+
+def password():
+    return config['rabbit']['password']
+
+
+def virtual_host():
+    return config['rabbit']['virtual_host']

verifier/dbverifier.py

@@ -38,6 +38,7 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
     sys.path.insert(0, POSSIBLE_TOPDIR)
 
 from stacktach import stacklog
+from verifier import config as verifier_config
 
 stacklog.set_default_logger_name('verifier')
 LOG = stacklog.get_logger()
@@ -333,34 +334,32 @@ def _create_exchange(name, type, exclusive=False, auto_delete=False,
                                  auto_delete=exclusive, durable=durable)
 
 
-def _create_connection(config):
+def _create_connection():
-    rabbit = config['rabbit']
+    conn_params = dict(hostname=verifier_config.host(),
-    conn_params = dict(hostname=rabbit['host'],
+                       port=verifier_config.port(),
-                       port=rabbit['port'],
+                       userid=verifier_config.userid(),
-                       userid=rabbit['userid'],
+                       password=verifier_config.password(),
-                       password=rabbit['password'],
                        transport="librabbitmq",
-                       virtual_host=rabbit['virtual_host'])
+                       virtual_host=verifier_config.virtual_host())
     return kombu.connection.BrokerConnection(**conn_params)
 
 
 class Verifier(object):
 
-    def __init__(self, config, pool=None, rec=None):
+    def __init__(self, exchange, pool=None, rec=None):
-        self.config = config
+        self.pool = pool or multiprocessing.Pool(verifier_config.pool_size())
-        self.pool = pool or multiprocessing.Pool(self.config['pool_size'])
+        self.reconcile = verifier_config.reconcile()
-        self.reconcile = self.config.get('reconcile', False)
+        self.reconciler = self._load_reconciler(rec=rec)
-        self.reconciler = self._load_reconciler(config, rec=rec)
         self.results = []
         self.failed = []
+        self.exchange = exchange
 
-    def _load_reconciler(self, config, rec=None):
+    def _load_reconciler(self, rec=None):
         if rec:
             return rec
 
         if self.reconcile:
-            config_loc = config.get('reconciler_config',
+            config_loc = verifier_config.reconciler_config()
-                                    '/etc/stacktach/reconciler_config.json')
             with open(config_loc, 'r') as rec_config_file:
                 rec_config = json.load(rec_config_file)
                 return reconciler.Reconciler(rec_config)
@@ -421,9 +420,9 @@ class Verifier(object):
         return datetime.datetime.utcnow()
 
     def _run(self, callback=None):
-        tick_time = self.config['tick_time']
+        tick_time = verifier_config.tick_time()
-        settle_units = self.config['settle_units']
+        settle_units = verifier_config.settle_units()
-        settle_time = self.config['settle_time']
+        settle_time = verifier_config.settle_time()
         while self._keep_running():
             with transaction.commit_on_success():
                 now = self._utcnow()
@@ -439,15 +438,13 @@ class Verifier(object):
             time.sleep(tick_time)
 
     def run(self):
-        if self.config['enable_notifications']:
+        if verifier_config.enable_notifications():
-            exchange = _create_exchange(self.config['rabbit']['exchange_name'],
+            exchange = _create_exchange(
-                                        'topic',
+                self.exchange, 'topic',
-                                        durable=self.config['rabbit']['durable_queue'])
+                durable=verifier_config.durable_queue())
-            routing_keys = None
+            routing_keys = verifier_config.source_topics()
-            if self.config['rabbit'].get('routing_keys') is not None:
-                routing_keys = self.config['rabbit']['routing_keys']
 
-            with _create_connection(self.config) as conn:
+            with _create_connection() as conn:
                 def callback(result):
                     (verified, exist) = result
                     if verified:
@@ -459,9 +456,9 @@ class Verifier(object):
             self._run()
 
     def _run_once(self, callback=None):
-        tick_time = self.config['tick_time']
+        tick_time = verifier_config.tick_time()
-        settle_units = self.config['settle_units']
+        settle_units = verifier_config.settle_units()
-        settle_time = self.config['settle_time']
+        settle_time = verifier_config.settle_time()
         now = self._utcnow()
         kwargs = {settle_units: settle_time}
         ending_max = now - datetime.timedelta(**kwargs)
@@ -475,15 +472,13 @@ class Verifier(object):
             time.sleep(tick_time)
 
     def run_once(self):
-        if self.config['enable_notifications']:
+        if verifier_config.enable_notifications():
-            exchange = _create_exchange(self.config['rabbit']['exchange_name'],
+            exchange = _create_exchange(
-                                        'topic',
+                self.exchange, 'topic',
-                                        durable=self.config['rabbit']['durable_queue'])
+                durable=verifier_config.durable_queue())
-            routing_keys = None
+            routing_keys = verifier_config.source_topics()[self.exchange]
-            if self.config['rabbit'].get('routing_keys') is not None:
-                routing_keys = self.config['rabbit']['routing_keys']
 
-            with _create_connection(self.config) as conn:
+            with _create_connection() as conn:
                 def callback(result):
                     (verified, exist) = result
                     if verified:
@@ -522,7 +517,7 @@ if __name__ == '__main__':
     config = {'tick_time': args.tick_time, 'settle_time': args.settle_time,
               'settle_units': args.settle_units, 'pool_size': args.pool_size}
 
-    verifier = Verifier(config)
+    verifier = Verifier('nova')
     if args.run_once:
         verifier.run_once()
     else:

verifier/start_verifier.py

@@ -31,9 +31,8 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
     sys.path.insert(0, POSSIBLE_TOPDIR)
 
 from verifier import dbverifier
+import verifier.config as verifier_config
 
-config_filename = os.environ.get('STACKTACH_VERIFIER_CONFIG',
-                                 'stacktach_verifier_config.json')
 try:
     from local_settings import *
     config_filename = STACKTACH_VERIFIER_CONFIG
@@ -55,17 +54,14 @@ def kill_time(signal, frame):
 
 
 if __name__ == '__main__':
-    config = None
+    def make_and_start_verifier(exchange):
-    with open(config_filename, "r") as f:
-        config = json.load(f)
-
-    def make_and_start_verifier(config):
         # Gotta create it and run it this way so things don't get
         # lost when the process is forked.
-        verifier = dbverifier.Verifier(config)
+        verifier = dbverifier.Verifier(exchange)
         verifier.run()
 
-    process = Process(target=make_and_start_verifier, args=(config,))
+    for exchange in verifier_config.source_topics().keys():
+        process = Process(target=make_and_start_verifier, args=(exchange,))
         process.start()
         signal.signal(signal.SIGINT, kill_time)
         signal.signal(signal.SIGTERM, kill_time)