x/stacktach
Code Issues Proposed changes

Refactored verifier config into a separate module

Browse Source
This commit is contained in:
Anuj Mathur 2013-07-16 17:43:57 +05:30
parent ca55663e9b
commit 3bee7db6f3
6 changed files with 186 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
stacktach/views.py
View File

@ -288,6 +288,7 @@ def _process_exists(raw, notification):
def _process_glance_usage(raw, notification):
notification.save_usage(raw)
def _process_glance_delete(raw, notification):
notification.save_delete(raw)

94
tests/unit/test_verifier_db.py
View File

@ -46,11 +46,16 @@ from utils import OS_VERSION_2
from utils import TENANT_ID_1
from utils import TENANT_ID_2
from utils import INSTANCE_TYPE_ID_1
from utils import TICK_TIME
from utils import SETTLE_TIME
from utils import SETTLE_UNITS
from verifier import dbverifier
from verifier import config as verifier_config
from verifier import AmbiguousResults
from verifier import FieldMismatch
from verifier import NotFound
from verifier import VerificationException
import stubout
class VerifierTestCase(unittest.TestCase):
@ -83,40 +88,28 @@ class VerifierTestCase(unittest.TestCase):
models.JsonReport.objects = self.mox.CreateMockAnything()
self._setup_verifier()
self.stubs = stubout.StubOutForTesting()
self.stubs.Set(verifier_config, 'durable_queue', lambda: True)
self.stubs.Set(verifier_config, 'source_topics', lambda:
{'nova': ['notifications.info']})
self.stubs.Set(verifier_config, 'enable_notifications', lambda: True)
self.stubs.Set(verifier_config, 'host', lambda: '10.0.0.1')
self.stubs.Set(verifier_config, 'virtual_host', lambda: '/')
self.stubs.Set(verifier_config, 'port', lambda: '5672')
self.stubs.Set(verifier_config, 'userid', lambda: 'rabbit')
self.stubs.Set(verifier_config, 'password', lambda: 'password')
self.stubs.Set(verifier_config, 'tick_time', lambda: TICK_TIME)
self.stubs.Set(verifier_config, 'settle_time', lambda: SETTLE_TIME)
self.stubs.Set(verifier_config, 'settle_units', lambda: SETTLE_UNITS)
def _setup_verifier(self):
self.config = {
"tick_time": 30,
"settle_time": 5,
"settle_units": "minutes",
"pool_size": 2,
"enable_notifications": False,
}
self.pool = self.mox.CreateMockAnything()
self.reconciler = self.mox.CreateMockAnything()
self.verifier = dbverifier.Verifier(self.config,
pool=self.pool,
self.verifier = dbverifier.Verifier('nova', pool=self.pool,
rec=self.reconciler)
self.config_notif = {
"tick_time": 30,
"settle_time": 5,
"settle_units": "minutes",
"pool_size": 2,
"enable_notifications": True,
"rabbit": {
"durable_queue": False,
"host": "10.0.0.1",
"port": 5672,
"userid": "rabbit",
"password": "rabbit",
"virtual_host": "/",
"exchange_name": "stacktach",
}
}
self.pool_notif = self.mox.CreateMockAnything()
self.reconciler_notif = self.mox.CreateMockAnything()
self.verifier_notif = dbverifier.Verifier(self.config_notif,
pool=self.pool_notif,
self.verifier_notif = dbverifier.Verifier('nova', pool=self.pool_notif,
rec=self.reconciler)
def tearDown(self):
@ -125,6 +118,7 @@ class VerifierTestCase(unittest.TestCase):
self.pool = None
self.verifier_notif = None
self.pool_notif = None
self.stubs.UnsetAll()
def test_verify_for_launch(self):
exist = self.mox.CreateMockAnything()
@ -1024,13 +1018,14 @@ class VerifierTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_run_notifications(self):
self.stubs.Set(verifier_config, 'durable_queue', lambda: False)
self.mox.StubOutWithMock(dbverifier, '_create_exchange')
exchange = self.mox.CreateMockAnything()
dbverifier._create_exchange('stacktach', 'topic', durable=False)\
dbverifier._create_exchange('nova', 'topic', durable=False)\
.AndReturn(exchange)
self.mox.StubOutWithMock(dbverifier, '_create_connection')
conn = self.mox.CreateMockAnything()
dbverifier._create_connection(self.config_notif).AndReturn(conn)
dbverifier._create_connection().AndReturn(conn)
conn.__enter__().AndReturn(conn)
self.mox.StubOutWithMock(self.verifier_notif, '_run')
self.verifier_notif._run(callback=mox.Not(mox.Is(None)))
@ -1040,13 +1035,14 @@ class VerifierTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_run_notifications_with_routing_keys(self):
self.stubs.Set(verifier_config, 'durable_queue', lambda: False)
self.mox.StubOutWithMock(dbverifier, '_create_exchange')
exchange = self.mox.CreateMockAnything()
dbverifier._create_exchange('stacktach', 'topic', durable=False) \
dbverifier._create_exchange('nova', 'topic', durable=False) \
.AndReturn(exchange)
self.mox.StubOutWithMock(dbverifier, '_create_connection')
conn = self.mox.CreateMockAnything()
dbverifier._create_connection(self.config_notif).AndReturn(conn)
dbverifier._create_connection().AndReturn(conn)
conn.__enter__().AndReturn(conn)
self.mox.StubOutWithMock(self.verifier_notif, '_run')
self.verifier_notif._run(callback=mox.Not(mox.Is(None)))
@ -1056,6 +1052,7 @@ class VerifierTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_run_no_notifications(self):
self.stubs.Set(verifier_config, 'enable_notifications', lambda: False)
self.mox.StubOutWithMock(self.verifier, '_run')
self.verifier._run()
self.mox.ReplayAll()
@ -1063,13 +1060,14 @@ class VerifierTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_run_once_notifications(self):
self.stubs.Set(verifier_config, 'durable_queue', lambda: False)
self.mox.StubOutWithMock(dbverifier, '_create_exchange')
exchange = self.mox.CreateMockAnything()
dbverifier._create_exchange('stacktach', 'topic', durable=False) \
dbverifier._create_exchange('nova', 'topic', durable=False) \
.AndReturn(exchange)
self.mox.StubOutWithMock(dbverifier, '_create_connection')
conn = self.mox.CreateMockAnything()
dbverifier._create_connection(self.config_notif).AndReturn(conn)
dbverifier._create_connection().AndReturn(conn)
conn.__enter__().AndReturn(conn)
self.mox.StubOutWithMock(self.verifier_notif, '_run_once')
self.verifier_notif._run_once(callback=mox.Not(mox.Is(None)))
@ -1079,6 +1077,8 @@ class VerifierTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_run_once_no_notifications(self):
self.mox.StubOutWithMock(verifier_config, 'enable_notifications')
verifier_config.enable_notifications().AndReturn(False)
self.mox.StubOutWithMock(self.verifier, '_run_once')
self.verifier._run_once()
self.mox.ReplayAll()
@ -1086,6 +1086,7 @@ class VerifierTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_run_full_no_notifications(self):
self.stubs.Set(verifier_config, 'enable_notifications', lambda: False)
self.verifier.reconcile = True
self.mox.StubOutWithMock(self.verifier, '_keep_running')
self.verifier._keep_running().AndReturn(True)
@ -1096,9 +1097,7 @@ class VerifierTestCase(unittest.TestCase):
start = datetime.datetime.utcnow()
self.mox.StubOutWithMock(self.verifier, '_utcnow')
self.verifier._utcnow().AndReturn(start)
settle_time = self.config['settle_time']
settle_units = self.config['settle_units']
settle_offset = {settle_units: settle_time}
settle_offset = {SETTLE_UNITS: SETTLE_TIME}
ending_max = start - datetime.timedelta(**settle_offset)
self.mox.StubOutWithMock(self.verifier, 'verify_for_range')
self.verifier.verify_for_range(ending_max, callback=None)
@ -1115,7 +1114,7 @@ class VerifierTestCase(unittest.TestCase):
self.verifier.reconcile_failed()
fake_transaction.__exit__(None, None, None)
self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
time.sleep(self.config['tick_time'])
time.sleep(TICK_TIME)
self.verifier._keep_running().AndReturn(False)
self.mox.ReplayAll()
self.verifier.run()
@ -1132,9 +1131,7 @@ class VerifierTestCase(unittest.TestCase):
start = datetime.datetime.utcnow()
self.mox.StubOutWithMock(self.verifier_notif, '_utcnow')
self.verifier_notif._utcnow().AndReturn(start)
settle_time = self.config['settle_time']
settle_units = self.config['settle_units']
settle_offset = {settle_units: settle_time}
settle_offset = {SETTLE_UNITS: SETTLE_TIME}
ending_max = start - datetime.timedelta(**settle_offset)
self.mox.StubOutWithMock(self.verifier_notif, 'verify_for_range')
self.verifier_notif.verify_for_range(ending_max,
@ -1152,20 +1149,19 @@ class VerifierTestCase(unittest.TestCase):
self.verifier_notif.reconcile_failed()
fake_transaction.__exit__(None, None, None)
self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
time.sleep(self.config['tick_time'])
time.sleep(TICK_TIME)
self.verifier_notif._keep_running().AndReturn(False)
self.mox.ReplayAll()
self.verifier_notif.run()
self.mox.VerifyAll()
def test_run_once_full_no_notifications(self):
self.stubs.Set(verifier_config, 'enable_notifications', lambda: False)
self.verifier.reconcile = True
start = datetime.datetime.utcnow()
self.mox.StubOutWithMock(self.verifier, '_utcnow')
self.verifier._utcnow().AndReturn(start)
settle_time = self.config['settle_time']
settle_units = self.config['settle_units']
settle_offset = {settle_units: settle_time}
settle_offset = {SETTLE_UNITS: SETTLE_TIME}
ending_max = start - datetime.timedelta(**settle_offset)
self.mox.StubOutWithMock(self.verifier, 'verify_for_range')
self.verifier.verify_for_range(ending_max, callback=None)
@ -1181,7 +1177,7 @@ class VerifierTestCase(unittest.TestCase):
self.mox.StubOutWithMock(self.verifier, 'reconcile_failed')
self.verifier.reconcile_failed()
self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
time.sleep(self.config['tick_time'])
time.sleep(TICK_TIME)
self.mox.ReplayAll()
self.verifier.run_once()
self.mox.VerifyAll()
@ -1191,9 +1187,7 @@ class VerifierTestCase(unittest.TestCase):
start = datetime.datetime.utcnow()
self.mox.StubOutWithMock(self.verifier_notif, '_utcnow')
self.verifier_notif._utcnow().AndReturn(start)
settle_time = self.config['settle_time']
settle_units = self.config['settle_units']
settle_offset = {settle_units: settle_time}
settle_offset = {SETTLE_UNITS: SETTLE_TIME}
ending_max = start - datetime.timedelta(**settle_offset)
self.mox.StubOutWithMock(self.verifier_notif, 'verify_for_range')
self.verifier_notif.verify_for_range(ending_max,
@ -1210,7 +1204,7 @@ class VerifierTestCase(unittest.TestCase):
self.mox.StubOutWithMock(self.verifier_notif, 'reconcile_failed')
self.verifier_notif.reconcile_failed()
self.mox.StubOutWithMock(time, 'sleep', use_mock_anything=True)
time.sleep(self.config['tick_time'])
time.sleep(TICK_TIME)
self.mox.ReplayAll()
self.verifier_notif.run_once()
self.mox.VerifyAll()

4
tests/unit/utils.py
View File

@ -56,6 +56,10 @@ OS_VERSION_1 = "1"
OS_VERSION_2 = "2"
TIMESTAMP_1 = "2013-06-20 17:31:57.939614"
SETTLE_TIME = 5
SETTLE_UNITS = "minutes"
TICK_TIME = 10
def decimal_utc(t = datetime.datetime.utcnow()):
return dt.dt_to_decimal(t)

95
verifier/config.py Normal file
View File

@ -0,0 +1,95 @@
# Copyright (c) 2013 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import json
import os
config_filename = os.environ.get('STACKTACH_VERIFIER_CONFIG',
'stacktach_verifier_config.json')
try:
from local_settings import *
config_filename = STACKTACH_VERIFIER_CONFIG
except ImportError:
pass
config = None
with open(config_filename, "r") as f:
config = json.load(f)
def enable_notifications():
return config['enable_notifications']
def source_topics():
return config['rabbit']['source_topics']
def tick_time():
return config['tick_time']
def settle_units():
return config['settle_units']
def settle_time():
return config['settle_time']
def reconcile():
return config.get('reconcile', False)
def reconciler_config():
return config.get(
'reconciler_config', '/etc/stacktach/reconciler_config.json')
def source_topics():
return config['rabbit']['source_topics']
def pool_size():
return config['pool_size']
def durable_queue():
return config['rabbit']['durable_queue']
def host():
return config['rabbit']['host']
def port():
return config['rabbit']['port']
def userid():
return config['rabbit']['userid']
def password():
return config['rabbit']['password']
def virtual_host():
return config['rabbit']['virtual_host']

71
verifier/dbverifier.py
View File

@ -38,6 +38,7 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
sys.path.insert(0, POSSIBLE_TOPDIR)
from stacktach import stacklog
from verifier import config as verifier_config
stacklog.set_default_logger_name('verifier')
LOG = stacklog.get_logger()
@ -333,34 +334,32 @@ def _create_exchange(name, type, exclusive=False, auto_delete=False,
auto_delete=exclusive, durable=durable)
def _create_connection(config):
rabbit = config['rabbit']
conn_params = dict(hostname=rabbit['host'],
port=rabbit['port'],
userid=rabbit['userid'],
password=rabbit['password'],
def _create_connection():
conn_params = dict(hostname=verifier_config.host(),
port=verifier_config.port(),
userid=verifier_config.userid(),
password=verifier_config.password(),
transport="librabbitmq",
virtual_host=rabbit['virtual_host'])
virtual_host=verifier_config.virtual_host())
return kombu.connection.BrokerConnection(**conn_params)
class Verifier(object):
def __init__(self, config, pool=None, rec=None):
self.config = config
self.pool = pool or multiprocessing.Pool(self.config['pool_size'])
self.reconcile = self.config.get('reconcile', False)
self.reconciler = self._load_reconciler(config, rec=rec)
def __init__(self, exchange, pool=None, rec=None):
self.pool = pool or multiprocessing.Pool(verifier_config.pool_size())
self.reconcile = verifier_config.reconcile()
self.reconciler = self._load_reconciler(rec=rec)
self.results = []
self.failed = []
self.exchange = exchange
def _load_reconciler(self, config, rec=None):
def _load_reconciler(self, rec=None):
if rec:
return rec
if self.reconcile:
config_loc = config.get('reconciler_config',
'/etc/stacktach/reconciler_config.json')
config_loc = verifier_config.reconciler_config()
with open(config_loc, 'r') as rec_config_file:
rec_config = json.load(rec_config_file)
return reconciler.Reconciler(rec_config)
@ -421,9 +420,9 @@ class Verifier(object):
return datetime.datetime.utcnow()
def _run(self, callback=None):
tick_time = self.config['tick_time']
settle_units = self.config['settle_units']
settle_time = self.config['settle_time']
tick_time = verifier_config.tick_time()
settle_units = verifier_config.settle_units()
settle_time = verifier_config.settle_time()
while self._keep_running():
with transaction.commit_on_success():
now = self._utcnow()
@ -439,15 +438,13 @@ class Verifier(object):
time.sleep(tick_time)
def run(self):
if self.config['enable_notifications']:
exchange = _create_exchange(self.config['rabbit']['exchange_name'],
'topic',
durable=self.config['rabbit']['durable_queue'])
routing_keys = None
if self.config['rabbit'].get('routing_keys') is not None:
routing_keys = self.config['rabbit']['routing_keys']
if verifier_config.enable_notifications():
exchange = _create_exchange(
self.exchange, 'topic',
durable=verifier_config.durable_queue())
routing_keys = verifier_config.source_topics()
with _create_connection(self.config) as conn:
with _create_connection() as conn:
def callback(result):
(verified, exist) = result
if verified:
@ -459,9 +456,9 @@ class Verifier(object):
self._run()
def _run_once(self, callback=None):
tick_time = self.config['tick_time']
settle_units = self.config['settle_units']
settle_time = self.config['settle_time']
tick_time = verifier_config.tick_time()
settle_units = verifier_config.settle_units()
settle_time = verifier_config.settle_time()
now = self._utcnow()
kwargs = {settle_units: settle_time}
ending_max = now - datetime.timedelta(**kwargs)
@ -475,15 +472,13 @@ class Verifier(object):
time.sleep(tick_time)
def run_once(self):
if self.config['enable_notifications']:
exchange = _create_exchange(self.config['rabbit']['exchange_name'],
'topic',
durable=self.config['rabbit']['durable_queue'])
routing_keys = None
if self.config['rabbit'].get('routing_keys') is not None:
routing_keys = self.config['rabbit']['routing_keys']
if verifier_config.enable_notifications():
exchange = _create_exchange(
self.exchange, 'topic',
durable=verifier_config.durable_queue())
routing_keys = verifier_config.source_topics()[self.exchange]
with _create_connection(self.config) as conn:
with _create_connection() as conn:
def callback(result):
(verified, exist) = result
if verified:
@ -522,7 +517,7 @@ if __name__ == '__main__':
config = {'tick_time': args.tick_time, 'settle_time': args.settle_time,
'settle_units': args.settle_units, 'pool_size': args.pool_size}
verifier = Verifier(config)
verifier = Verifier('nova')
if args.run_once:
verifier.run_once()
else:

14
verifier/start_verifier.py
View File

@ -31,9 +31,8 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
sys.path.insert(0, POSSIBLE_TOPDIR)
from verifier import dbverifier
import verifier.config as verifier_config
config_filename = os.environ.get('STACKTACH_VERIFIER_CONFIG',
'stacktach_verifier_config.json')
try:
from local_settings import *
config_filename = STACKTACH_VERIFIER_CONFIG
@ -55,17 +54,14 @@ def kill_time(signal, frame):
if __name__ == '__main__':
config = None
with open(config_filename, "r") as f:
config = json.load(f)
def make_and_start_verifier(config):
def make_and_start_verifier(exchange):
# Gotta create it and run it this way so things don't get
# lost when the process is forked.
verifier = dbverifier.Verifier(config)
verifier = dbverifier.Verifier(exchange)
verifier.run()
process = Process(target=make_and_start_verifier, args=(config,))
for exchange in verifier_config.source_topics().keys():
process = Process(target=make_and_start_verifier, args=(exchange,))
process.start()
signal.signal(signal.SIGINT, kill_time)
signal.signal(signal.SIGTERM, kill_time)