From 3bee7db6f32dda5ce4fcdf9db5b6fd4ed5eab4aa Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Tue, 16 Jul 2013 17:43:57 +0530 Subject: [PATCH] Refactored verifier config into a separate module --- stacktach/views.py | 1 + tests/unit/test_verifier_db.py | 94 ++++++++++++++++----------------- tests/unit/utils.py | 4 ++ verifier/config.py | 95 ++++++++++++++++++++++++++++++++++ verifier/dbverifier.py | 71 ++++++++++++------------- verifier/start_verifier.py | 22 ++++---- 6 files changed, 186 insertions(+), 101 deletions(-) create mode 100644 verifier/config.py diff --git a/stacktach/views.py b/stacktach/views.py index 14c5d08..a6699bf 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -288,6 +288,7 @@ def _process_exists(raw, notification): def _process_glance_usage(raw, notification): notification.save_usage(raw) + def _process_glance_delete(raw, notification): notification.save_delete(raw) diff --git a/tests/unit/test_verifier_db.py b/tests/unit/test_verifier_db.py index 795528b..571a5b6 100644 --- a/tests/unit/test_verifier_db.py +++ b/tests/unit/test_verifier_db.py @@ -46,11 +46,16 @@ from utils import OS_VERSION_2 from utils import TENANT_ID_1 from utils import TENANT_ID_2 from utils import INSTANCE_TYPE_ID_1 +from utils import TICK_TIME +from utils import SETTLE_TIME +from utils import SETTLE_UNITS from verifier import dbverifier +from verifier import config as verifier_config from verifier import AmbiguousResults from verifier import FieldMismatch from verifier import NotFound from verifier import VerificationException +import stubout class VerifierTestCase(unittest.TestCase): @@ -83,40 +88,28 @@ class VerifierTestCase(unittest.TestCase): models.JsonReport.objects = self.mox.CreateMockAnything() self._setup_verifier() + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(verifier_config, 'durable_queue', lambda: True) + self.stubs.Set(verifier_config, 'source_topics', lambda: + {'nova': ['notifications.info']}) + self.stubs.Set(verifier_config, 'enable_notifications', lambda: True) + self.stubs.Set(verifier_config, 'host', lambda: '10.0.0.1') + self.stubs.Set(verifier_config, 'virtual_host', lambda: '/') + self.stubs.Set(verifier_config, 'port', lambda: '5672') + self.stubs.Set(verifier_config, 'userid', lambda: 'rabbit') + self.stubs.Set(verifier_config, 'password', lambda: 'password') + self.stubs.Set(verifier_config, 'tick_time', lambda: TICK_TIME) + self.stubs.Set(verifier_config, 'settle_time', lambda: SETTLE_TIME) + self.stubs.Set(verifier_config, 'settle_units', lambda: SETTLE_UNITS) + def _setup_verifier(self): - self.config = { - "tick_time": 30, - "settle_time": 5, - "settle_units": "minutes", - "pool_size": 2, - "enable_notifications": False, - } self.pool = self.mox.CreateMockAnything() self.reconciler = self.mox.CreateMockAnything() - self.verifier = dbverifier.Verifier(self.config, - pool=self.pool, + self.verifier = dbverifier.Verifier('nova', pool=self.pool, rec=self.reconciler) - - self.config_notif = { - "tick_time": 30, - "settle_time": 5, - "settle_units": "minutes", - "pool_size": 2, - "enable_notifications": True, - "rabbit": { - "durable_queue": False, - "host": "10.0.0.1", - "port": 5672, - "userid": "rabbit", - "password": "rabbit", - "virtual_host": "/", - "exchange_name": "stacktach", - } - } self.pool_notif = self.mox.CreateMockAnything() self.reconciler_notif = self.mox.CreateMockAnything() - self.verifier_notif = dbverifier.Verifier(self.config_notif, - pool=self.pool_notif, + self.verifier_notif = dbverifier.Verifier('nova', pool=self.pool_notif, rec=self.reconciler) def tearDown(self): @@ -125,6 +118,7 @@ class VerifierTestCase(unittest.TestCase): self.pool = None self.verifier_notif = None self.pool_notif = None + self.stubs.UnsetAll() def test_verify_for_launch(self): exist = self.mox.CreateMockAnything() @@ -1024,13 +1018,14 @@ class VerifierTestCase(unittest.TestCase): self.mox.VerifyAll() def test_run_notifications(self): + self.stubs.Set(verifier_config, 'durable_queue', lambda: False) self.mox.StubOutWithMock(dbverifier, '_create_exchange') exchange = self.mox.CreateMockAnything() - dbverifier._create_exchange('stacktach', 'topic', durable=False)\ + dbverifier._create_exchange('nova', 'topic', durable=False)\ .AndReturn(exchange) self.mox.StubOutWithMock(dbverifier, '_create_connection') conn = self.mox.CreateMockAnything() - dbverifier._create_connection(self.config_notif).AndReturn(conn) + dbverifier._create_connection().AndReturn(conn) conn.__enter__().AndReturn(conn) self.mox.StubOutWithMock(self.verifier_notif, '_run') self.verifier_notif._run(callback=mox.Not(mox.Is(None))) @@ -1040,13 +1035,14 @@ class VerifierTestCase(unittest.TestCase): self.mox.VerifyAll() def test_run_notifications_with_routing_keys(self): + self.stubs.Set(verifier_config, 'durable_queue', lambda: False) self.mox.StubOutWithMock(dbverifier, '_create_exchange') exchange = self.mox.CreateMockAnything() - dbverifier._create_exchange('stacktach', 'topic', durable=False) \ + dbverifier._create_exchange('nova', 'topic', durable=False) \ .AndReturn(exchange) self.mox.StubOutWithMock(dbverifier, '_create_connection') conn = self.mox.CreateMockAnything() - dbverifier._create_connection(self.config_notif).AndReturn(conn) + dbverifier._create_connection().AndReturn(conn) conn.__enter__().AndReturn(conn) self.mox.StubOutWithMock(self.verifier_notif, '_run') self.verifier_notif._run(callback=mox.Not(mox.Is(None))) @@ -1056,6 +1052,7 @@ class VerifierTestCase(unittest.TestCase): self.mox.VerifyAll() def test_run_no_notifications(self): + self.stubs.Set(verifier_config, 'enable_notifications', lambda: False) self.mox.StubOutWithMock(self.verifier, '_run') self.verifier._run() self.mox.ReplayAll() @@ -1063,13 +1060,14 @@ class VerifierTestCase(unittest.TestCase): self.mox.VerifyAll() def test_run_once_notifications(self): + self.stubs.Set(verifier_config, 'durable_queue', lambda: False) self.mox.StubOutWithMock(dbverifier, '_create_exchange') exchange = self.mox.CreateMockAnything() - dbverifier._create_exchange('stacktach', 'topic', durable=False) \ + dbverifier._create_exchange('nova', 'topic', durable=False) \ .AndReturn(exchange) self.mox.StubOutWithMock(dbverifier, '_create_connection') conn = self.mox.CreateMockAnything() - dbverifier._create_connection(self.config_notif).AndReturn(conn) + dbverifier._create_connection().AndReturn(conn) conn.__enter__().AndReturn(conn) self.mox.StubOutWithMock(self.verifier_notif, '_run_once') self.verifier_notif._run_once(callback=mox.Not(mox.Is(None))) @@ -1079,6 +1077,8 @@ class VerifierTestCase(unittest.TestCase): self.mox.VerifyAll() def test_run_once_no_notifications(self): + self.mox.StubOutWithMock(verifier_config, 'enable_notifications') + verifier_config.enable_notifications().AndReturn(False) self.mox.StubOutWithMock(self.verifier, '_run_once') self.verifier._run_once() self.mox.ReplayAll() @@ -1086,6 +1086,7 @@ class VerifierTestCase(unittest.TestCase): self.mox.VerifyAll() def test_run_full_no_notifications(self): + self.stubs.Set(verifier_config, 'enable_notifications', lambda: False) self.verifier.reconcile = True self.mox.StubOutWithMock(self.verifier, '_keep_running') self.verifier._keep_running().AndReturn(True) @@ -1096,9 +1097,7 @@ class VerifierTestCase(unittest.TestCase): start = datetime.datetime.utcnow() self.mox.StubOutWithMock(self.verifier, '_utcnow') self.verifier._utcnow().AndReturn(start) - settle_time = self.config['settle_time'] - settle_units = self.config['settle_units'] - settle_offset = {settle_units: settle_time} + settle_offset = {SETTLE_UNITS: SETTLE_TIME} ending_max = start - datetime.timedelta(**settle_offset) self.mox.StubOutWithMock(self.verifier, 'verify_for_range') self.verifier.verify_for_range(ending_max, callback=None) @@ -1115,7 +1114,7 @@ class VerifierTestCase(unittest.TestCase): self.verifier.reconcile_failed() fake_transaction.__exit__(None, None, None) self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True) - time.sleep(self.config['tick_time']) + time.sleep(TICK_TIME) self.verifier._keep_running().AndReturn(False) self.mox.ReplayAll() self.verifier.run() @@ -1132,9 +1131,7 @@ class VerifierTestCase(unittest.TestCase): start = datetime.datetime.utcnow() self.mox.StubOutWithMock(self.verifier_notif, '_utcnow') self.verifier_notif._utcnow().AndReturn(start) - settle_time = self.config['settle_time'] - settle_units = self.config['settle_units'] - settle_offset = {settle_units: settle_time} + settle_offset = {SETTLE_UNITS: SETTLE_TIME} ending_max = start - datetime.timedelta(**settle_offset) self.mox.StubOutWithMock(self.verifier_notif, 'verify_for_range') self.verifier_notif.verify_for_range(ending_max, @@ -1152,20 +1149,19 @@ class VerifierTestCase(unittest.TestCase): self.verifier_notif.reconcile_failed() fake_transaction.__exit__(None, None, None) self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True) - time.sleep(self.config['tick_time']) + time.sleep(TICK_TIME) self.verifier_notif._keep_running().AndReturn(False) self.mox.ReplayAll() self.verifier_notif.run() self.mox.VerifyAll() def test_run_once_full_no_notifications(self): + self.stubs.Set(verifier_config, 'enable_notifications', lambda: False) self.verifier.reconcile = True start = datetime.datetime.utcnow() self.mox.StubOutWithMock(self.verifier, '_utcnow') self.verifier._utcnow().AndReturn(start) - settle_time = self.config['settle_time'] - settle_units = self.config['settle_units'] - settle_offset = {settle_units: settle_time} + settle_offset = {SETTLE_UNITS: SETTLE_TIME} ending_max = start - datetime.timedelta(**settle_offset) self.mox.StubOutWithMock(self.verifier, 'verify_for_range') self.verifier.verify_for_range(ending_max, callback=None) @@ -1181,7 +1177,7 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(self.verifier, 'reconcile_failed') self.verifier.reconcile_failed() self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True) - time.sleep(self.config['tick_time']) + time.sleep(TICK_TIME) self.mox.ReplayAll() self.verifier.run_once() self.mox.VerifyAll() @@ -1191,9 +1187,7 @@ class VerifierTestCase(unittest.TestCase): start = datetime.datetime.utcnow() self.mox.StubOutWithMock(self.verifier_notif, '_utcnow') self.verifier_notif._utcnow().AndReturn(start) - settle_time = self.config['settle_time'] - settle_units = self.config['settle_units'] - settle_offset = {settle_units: settle_time} + settle_offset = {SETTLE_UNITS: SETTLE_TIME} ending_max = start - datetime.timedelta(**settle_offset) self.mox.StubOutWithMock(self.verifier_notif, 'verify_for_range') self.verifier_notif.verify_for_range(ending_max, @@ -1210,7 +1204,7 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(self.verifier_notif, 'reconcile_failed') self.verifier_notif.reconcile_failed() self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True) - time.sleep(self.config['tick_time']) + time.sleep(TICK_TIME) self.mox.ReplayAll() self.verifier_notif.run_once() self.mox.VerifyAll() diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 222b427..7ff23e7 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -56,6 +56,10 @@ OS_VERSION_1 = "1" OS_VERSION_2 = "2" TIMESTAMP_1 = "2013-06-20 17:31:57.939614" +SETTLE_TIME = 5 +SETTLE_UNITS = "minutes" +TICK_TIME = 10 + def decimal_utc(t = datetime.datetime.utcnow()): return dt.dt_to_decimal(t) diff --git a/verifier/config.py b/verifier/config.py new file mode 100644 index 0000000..b9ae82e --- /dev/null +++ b/verifier/config.py @@ -0,0 +1,95 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import json +import os + +config_filename = os.environ.get('STACKTACH_VERIFIER_CONFIG', + 'stacktach_verifier_config.json') + +try: + from local_settings import * + config_filename = STACKTACH_VERIFIER_CONFIG +except ImportError: + pass + +config = None +with open(config_filename, "r") as f: + config = json.load(f) + + +def enable_notifications(): + return config['enable_notifications'] + + +def source_topics(): + return config['rabbit']['source_topics'] + + +def tick_time(): + return config['tick_time'] + + +def settle_units(): + return config['settle_units'] + + +def settle_time(): + return config['settle_time'] + + +def reconcile(): + return config.get('reconcile', False) + + +def reconciler_config(): + return config.get( + 'reconciler_config', '/etc/stacktach/reconciler_config.json') + + +def source_topics(): + return config['rabbit']['source_topics'] + + +def pool_size(): + return config['pool_size'] + + +def durable_queue(): + return config['rabbit']['durable_queue'] + + +def host(): + return config['rabbit']['host'] + + +def port(): + return config['rabbit']['port'] + + +def userid(): + return config['rabbit']['userid'] + + +def password(): + return config['rabbit']['password'] + + +def virtual_host(): + return config['rabbit']['virtual_host'] \ No newline at end of file diff --git a/verifier/dbverifier.py b/verifier/dbverifier.py index 9361ff8..2092100 100644 --- a/verifier/dbverifier.py +++ b/verifier/dbverifier.py @@ -38,6 +38,7 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): sys.path.insert(0, POSSIBLE_TOPDIR) from stacktach import stacklog +from verifier import config as verifier_config stacklog.set_default_logger_name('verifier') LOG = stacklog.get_logger() @@ -333,34 +334,32 @@ def _create_exchange(name, type, exclusive=False, auto_delete=False, auto_delete=exclusive, durable=durable) -def _create_connection(config): - rabbit = config['rabbit'] - conn_params = dict(hostname=rabbit['host'], - port=rabbit['port'], - userid=rabbit['userid'], - password=rabbit['password'], +def _create_connection(): + conn_params = dict(hostname=verifier_config.host(), + port=verifier_config.port(), + userid=verifier_config.userid(), + password=verifier_config.password(), transport="librabbitmq", - virtual_host=rabbit['virtual_host']) + virtual_host=verifier_config.virtual_host()) return kombu.connection.BrokerConnection(**conn_params) class Verifier(object): - def __init__(self, config, pool=None, rec=None): - self.config = config - self.pool = pool or multiprocessing.Pool(self.config['pool_size']) - self.reconcile = self.config.get('reconcile', False) - self.reconciler = self._load_reconciler(config, rec=rec) + def __init__(self, exchange, pool=None, rec=None): + self.pool = pool or multiprocessing.Pool(verifier_config.pool_size()) + self.reconcile = verifier_config.reconcile() + self.reconciler = self._load_reconciler(rec=rec) self.results = [] self.failed = [] + self.exchange = exchange - def _load_reconciler(self, config, rec=None): + def _load_reconciler(self, rec=None): if rec: return rec if self.reconcile: - config_loc = config.get('reconciler_config', - '/etc/stacktach/reconciler_config.json') + config_loc = verifier_config.reconciler_config() with open(config_loc, 'r') as rec_config_file: rec_config = json.load(rec_config_file) return reconciler.Reconciler(rec_config) @@ -421,9 +420,9 @@ class Verifier(object): return datetime.datetime.utcnow() def _run(self, callback=None): - tick_time = self.config['tick_time'] - settle_units = self.config['settle_units'] - settle_time = self.config['settle_time'] + tick_time = verifier_config.tick_time() + settle_units = verifier_config.settle_units() + settle_time = verifier_config.settle_time() while self._keep_running(): with transaction.commit_on_success(): now = self._utcnow() @@ -439,15 +438,13 @@ class Verifier(object): time.sleep(tick_time) def run(self): - if self.config['enable_notifications']: - exchange = _create_exchange(self.config['rabbit']['exchange_name'], - 'topic', - durable=self.config['rabbit']['durable_queue']) - routing_keys = None - if self.config['rabbit'].get('routing_keys') is not None: - routing_keys = self.config['rabbit']['routing_keys'] + if verifier_config.enable_notifications(): + exchange = _create_exchange( + self.exchange, 'topic', + durable=verifier_config.durable_queue()) + routing_keys = verifier_config.source_topics() - with _create_connection(self.config) as conn: + with _create_connection() as conn: def callback(result): (verified, exist) = result if verified: @@ -459,9 +456,9 @@ class Verifier(object): self._run() def _run_once(self, callback=None): - tick_time = self.config['tick_time'] - settle_units = self.config['settle_units'] - settle_time = self.config['settle_time'] + tick_time = verifier_config.tick_time() + settle_units = verifier_config.settle_units() + settle_time = verifier_config.settle_time() now = self._utcnow() kwargs = {settle_units: settle_time} ending_max = now - datetime.timedelta(**kwargs) @@ -475,15 +472,13 @@ class Verifier(object): time.sleep(tick_time) def run_once(self): - if self.config['enable_notifications']: - exchange = _create_exchange(self.config['rabbit']['exchange_name'], - 'topic', - durable=self.config['rabbit']['durable_queue']) - routing_keys = None - if self.config['rabbit'].get('routing_keys') is not None: - routing_keys = self.config['rabbit']['routing_keys'] + if verifier_config.enable_notifications(): + exchange = _create_exchange( + self.exchange, 'topic', + durable=verifier_config.durable_queue()) + routing_keys = verifier_config.source_topics()[self.exchange] - with _create_connection(self.config) as conn: + with _create_connection() as conn: def callback(result): (verified, exist) = result if verified: @@ -522,7 +517,7 @@ if __name__ == '__main__': config = {'tick_time': args.tick_time, 'settle_time': args.settle_time, 'settle_units': args.settle_units, 'pool_size': args.pool_size} - verifier = Verifier(config) + verifier = Verifier('nova') if args.run_once: verifier.run_once() else: diff --git a/verifier/start_verifier.py b/verifier/start_verifier.py index aab3a29..d95cc23 100644 --- a/verifier/start_verifier.py +++ b/verifier/start_verifier.py @@ -31,9 +31,8 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): sys.path.insert(0, POSSIBLE_TOPDIR) from verifier import dbverifier +import verifier.config as verifier_config -config_filename = os.environ.get('STACKTACH_VERIFIER_CONFIG', - 'stacktach_verifier_config.json') try: from local_settings import * config_filename = STACKTACH_VERIFIER_CONFIG @@ -55,18 +54,15 @@ def kill_time(signal, frame): if __name__ == '__main__': - config = None - with open(config_filename, "r") as f: - config = json.load(f) - - def make_and_start_verifier(config): + def make_and_start_verifier(exchange): # Gotta create it and run it this way so things don't get # lost when the process is forked. - verifier = dbverifier.Verifier(config) + verifier = dbverifier.Verifier(exchange) verifier.run() - process = Process(target=make_and_start_verifier, args=(config,)) - process.start() - signal.signal(signal.SIGINT, kill_time) - signal.signal(signal.SIGTERM, kill_time) - signal.pause() + for exchange in verifier_config.source_topics().keys(): + process = Process(target=make_and_start_verifier, args=(exchange,)) + process.start() + signal.signal(signal.SIGINT, kill_time) + signal.signal(signal.SIGTERM, kill_time) + signal.pause()