x/stacktach
Code Issues Proposed changes

Fixing some usage bugs

Browse Source
This commit is contained in:
Andrew Melton 2013-05-28 14:13:18 -04:00
parent bb2861358e
commit 93fd8ad8b7
8 changed files with 375 additions and 65 deletions

35
stacktach/db.py

@ -1,4 +1,19 @@
import models
from stacktach import logging as stacklog
from stacktach import models
def _safe_get(Model, **kwargs):
object = None
query = Model.objects.filter(**kwargs)
count = query.count()
if count > 1:
stacklog.warn('Multiple records found for %s get.' % Model.__name__)
object = query[0]
elif count < 1:
stacklog.warn('No records found for %s get.' % Model.__name__)
else:
object = query[0]
return object
def get_or_create_deployment(name):
@ -41,13 +56,12 @@ def get_or_create_instance_usage(**kwargs):
return models.InstanceUsage.objects.get_or_create(**kwargs)
def get_or_create_instance_delete(**kwargs):
return models.InstanceDeletes.objects.get_or_create(**kwargs)
def get_instance_usage(**kwargs):
usage = None
try:
usage = models.InstanceUsage.objects.get(**kwargs)
except models.InstanceUsage.DoesNotExist:
pass
return usage
return _safe_get(models.InstanceUsage, **kwargs)
def create_instance_delete(**kwargs):
@ -55,12 +69,7 @@ def create_instance_delete(**kwargs):
def get_instance_delete(**kwargs):
delete = None
try:
delete = models.InstanceDeletes.objects.get(**kwargs)
except models.InstanceDeletes.DoesNotExist:
pass
return delete
return _safe_get(models.InstanceDeletes, **kwargs)
def create_instance_exists(**kwargs):

43
stacktach/logging.py Normal file

@ -0,0 +1,43 @@
# Copyright (c) 2013 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
LOG = None
def set_logger(logger):
global LOG
LOG = logger
def get_logger():
global LOG
return LOG
def warn(msg):
global LOG
if LOG is not None:
LOG.warn(msg)
def error(msg):
global LOG
if LOG is not None:
LOG.error(msg)

82
stacktach/views.py

@ -9,14 +9,23 @@ from django.shortcuts import render_to_response
from stacktach import datetime_to_decimal as dt
from stacktach import db as stackdb
from stacktach import models
from stacktach import image_type
from stacktach import models
from stacktach import logging as stacklog
from stacktach import utils
STACKDB = stackdb
def log_warn(msg):
global LOG
if LOG is None:
LOG = stacklog.get_logger()
if LOG is not None:
LOG.warn(msg)
def _extract_states(payload):
return {
'state' : payload.get('state', ""),
@ -278,53 +287,58 @@ def _process_delete(raw, body):
values = {
'instance': instance_id,
'deleted_at': deleted_at,
'raw': raw
}
delete = STACKDB.get_or_create_instance_delete(**values)
delete.raw = raw
launched_at = payload.get('launched_at')
if launched_at and launched_at != '':
launched_at = utils.str_time_to_unix(launched_at)
values['launched_at'] = launched_at
delete.launched_at = launched_at
delete = STACKDB.create_instance_delete(**values)
STACKDB.save(delete)
def _process_exists(raw, body):
payload = body['payload']
instance_id = payload['instance_id']
launched_at = utils.str_time_to_unix(payload['launched_at'])
launched_range = (launched_at, launched_at+1)
usage = STACKDB.get_instance_usage(instance=instance_id,
launched_at__range=launched_range)
values = {}
values['message_id'] = body['message_id']
values['instance'] = instance_id
values['launched_at'] = launched_at
beginning = utils.str_time_to_unix(payload['audit_period_beginning'])
values['audit_period_beginning'] = beginning
ending = utils.str_time_to_unix(payload['audit_period_ending'])
values['audit_period_ending'] = ending
values['instance_type_id'] = payload['instance_type_id']
if usage:
values['usage'] = usage
values['raw'] = raw
values['tenant'] = payload['tenant_id']
launched_at_str = payload.get('launched_at')
if launched_at_str is not None and launched_at_str != '':
launched_at = utils.str_time_to_unix(payload['launched_at'])
launched_range = (launched_at, launched_at+1)
usage = STACKDB.get_instance_usage(instance=instance_id,
launched_at__range=launched_range)
values = {}
values['message_id'] = body['message_id']
values['instance'] = instance_id
values['launched_at'] = launched_at
beginning = utils.str_time_to_unix(payload['audit_period_beginning'])
values['audit_period_beginning'] = beginning
ending = utils.str_time_to_unix(payload['audit_period_ending'])
values['audit_period_ending'] = ending
values['instance_type_id'] = payload['instance_type_id']
if usage:
values['usage'] = usage
values['raw'] = raw
values['tenant'] = payload['tenant_id']
deleted_at = payload.get('deleted_at')
if deleted_at and deleted_at != '':
# We only want to pre-populate the 'delete' if we know this is in fact
# an exist event for a deleted instance. Otherwise, there is a
# chance we may populate it for a previous period's exist.
delete = STACKDB.get_instance_delete(instance=instance_id,
launched_at__range=launched_range)
deleted_at = utils.str_time_to_unix(deleted_at)
values['deleted_at'] = deleted_at
if delete:
values['delete'] = delete
deleted_at = payload.get('deleted_at')
if deleted_at and deleted_at != '':
# We only want to pre-populate the 'delete' if we know this is in
# fact an exist event for a deleted instance. Otherwise, there
# is a chance we may populate it for a previous period's exist.
filter = {'instance': instance_id,
'launched_at__range': launched_range}
delete = STACKDB.get_instance_delete(**filter)
deleted_at = utils.str_time_to_unix(deleted_at)
values['deleted_at'] = deleted_at
if delete:
values['delete'] = delete
exists = STACKDB.create_instance_exists(**values)
STACKDB.save(exists)
exists = STACKDB.create_instance_exists(**values)
STACKDB.save(exists)
else:
stacklog.warn("Ignoring exists without launched_at. RawData(%s)" % raw.id)
USAGE_PROCESS_MAPPING = {

37
tests/unit/test_stacktach.py

@ -32,6 +32,7 @@ from utils import TENANT_ID_1
from utils import INSTANCE_TYPE_ID_1
from utils import DUMMY_TIME
from utils import INSTANCE_TYPE_ID_2
from stacktach import logging as stacklog
from stacktach import views
@ -380,7 +381,6 @@ class StacktachLifecycleTestCase(unittest.TestCase):
self.mox.VerifyAll()
def test_aggregate_lifecycle_update(self):
event = 'compute.instance.update'
when = datetime.datetime.utcnow()
@ -408,9 +408,12 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
views.STACKDB = self.mox.CreateMockAnything()
self.log = self.mox.CreateMockAnything()
stacklog.set_logger(self.log)
def tearDown(self):
self.mox.UnsetStubs()
stacklog.set_logger(None)
def test_process_usage_for_new_launch_create_start(self):
kwargs = {'launched': str(DUMMY_TIME), 'tenant_id': TENANT_ID_1 }
@ -595,10 +598,8 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
delete.instance = INSTANCE_ID_1
delete.launched_at = launch_decimal
delete.deleted_at = delete_decimal
views.STACKDB.create_instance_delete(instance=INSTANCE_ID_1,
launched_at=launch_decimal,
deleted_at=delete_decimal,
raw=raw)\
views.STACKDB.get_or_create_instance_delete(instance=INSTANCE_ID_1,
deleted_at=delete_decimal)\
.AndReturn(delete)
views.STACKDB.save(delete)
self.mox.ReplayAll()
@ -621,10 +622,9 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
delete = self.mox.CreateMockAnything()
delete.instance = INSTANCE_ID_1
delete.deleted_at = delete_decimal
views.STACKDB.create_instance_delete(instance=INSTANCE_ID_1,
deleted_at=delete_decimal,
raw=raw) \
.AndReturn(delete)
views.STACKDB.get_or_create_instance_delete(instance=INSTANCE_ID_1,
deleted_at=delete_decimal)\
.AndReturn(delete)
views.STACKDB.save(delete)
self.mox.ReplayAll()
@ -672,6 +672,23 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
views._process_exists(raw, notif[1])
self.mox.VerifyAll()
def test_process_exists_no_launched_at(self):
current_time = datetime.datetime.utcnow()
current_decimal = utils.decimal_utc(current_time)
audit_beginning = current_time - datetime.timedelta(hours=20)
notif = utils.create_nova_notif(audit_period_beginning=str(audit_beginning),
audit_period_ending=str(current_time),
tenant_id=TENANT_ID_1)
json_str = json.dumps(notif)
event = 'compute.instance.exists'
raw = utils.create_raw(self.mox, current_decimal, event=event,
json_str=json_str)
raw.id = 1
self.log.warn('Exists without launched_at. RawData(1)')
self.mox.ReplayAll()
views._process_exists(raw, notif[1])
self.mox.VerifyAll()
def test_process_exists_with_deleted_at(self):
current_time = datetime.datetime.utcnow()
launch_time = current_time - datetime.timedelta(hours=23)
@ -686,7 +703,7 @@ class StacktachUsageParsingTestCase(unittest.TestCase):
deleted=str(deleted_time),
audit_period_beginning=str(audit_beginning),
audit_period_ending=str(current_time),
tenant_id= TENANT_ID_1)
tenant_id=TENANT_ID_1)
json_str = json.dumps(notif)
event = 'compute.instance.exists'
raw = utils.create_raw(self.mox, current_decimal, event=event,

219
tests/unit/test_stacktach_db.py Normal file

@ -0,0 +1,219 @@
# Copyright (c) 2012 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import datetime
import unittest
import mox
from stacktach import db
from stacktach import logging as stacklog
from stacktach import models
class DBAPITestCase(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
self.log = self.mox.CreateMockAnything()
stacklog.set_logger(self.log)
self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True)
models.RawData.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True)
models.Deployment.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'Lifecycle', use_mock_anything=True)
models.Lifecycle.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'Timing', use_mock_anything=True)
models.Timing.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'RequestTracker',
use_mock_anything=True)
models.RequestTracker.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'InstanceUsage',
use_mock_anything=True)
models.InstanceUsage.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'InstanceDeletes',
use_mock_anything=True)
models.InstanceDeletes.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'InstanceExists',
use_mock_anything=True)
models.InstanceExists.objects = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models, 'JsonReport', use_mock_anything=True)
models.JsonReport.objects = self.mox.CreateMockAnything()
def tearDown(self):
self.mox.UnsetStubs()
stacklog.set_logger(None)
def test_safe_get(self):
Model = self.mox.CreateMockAnything()
Model.objects = self.mox.CreateMockAnything()
filters = {'field1': 'value1', 'field2': 'value2'}
results = self.mox.CreateMockAnything()
Model.objects.filter(**filters).AndReturn(results)
results.count().AndReturn(1)
object = self.mox.CreateMockAnything()
results[0].AndReturn(object)
self.mox.ReplayAll()
returned = db._safe_get(Model, **filters)
self.assertEqual(returned, object)
self.mox.VerifyAll()
def test_safe_get_no_results(self):
Model = self.mox.CreateMockAnything()
Model.__name__ = 'Model'
Model.objects = self.mox.CreateMockAnything()
filters = {'field1': 'value1', 'field2': 'value2'}
results = self.mox.CreateMockAnything()
Model.objects.filter(**filters).AndReturn(results)
results.count().AndReturn(0)
self.log.warn('No records found for Model get.')
self.mox.ReplayAll()
returned = db._safe_get(Model, **filters)
self.assertEqual(returned, None)
self.mox.VerifyAll()
def test_safe_get_multiple_results(self):
Model = self.mox.CreateMockAnything()
Model.__name__ = 'Model'
Model.objects = self.mox.CreateMockAnything()
filters = {'field1': 'value1', 'field2': 'value2'}
results = self.mox.CreateMockAnything()
Model.objects.filter(**filters).AndReturn(results)
results.count().AndReturn(2)
self.log.warn('Multiple records found for Model get.')
object = self.mox.CreateMockAnything()
results[0].AndReturn(object)
self.mox.ReplayAll()
returned = db._safe_get(Model, **filters)
self.assertEqual(returned, object)
self.mox.VerifyAll()
def test_get_or_create_deployment(self):
deployment = self.mox.CreateMockAnything()
models.Deployment.objects.get_or_create(name='test').AndReturn(deployment)
self.mox.ReplayAll()
returned = db.get_or_create_deployment('test')
self.assertEqual(returned, deployment)
self.mox.VerifyAll()
def _test_db_create_func(self, Model, func):
params = {'field1': 'value1', 'field2': 'value2'}
object = self.mox.CreateMockAnything()
Model(**params).AndReturn(object)
self.mox.ReplayAll()
returned = func(**params)
self.assertEqual(returned, object)
self.mox.VerifyAll()
def test_create_rawdata(self):
self._test_db_create_func(models.RawData, db.create_rawdata)
def test_create_lifecycle(self):
self._test_db_create_func(models.Lifecycle, db.create_lifecycle)
def test_create_timing(self):
self._test_db_create_func(models.Timing, db.create_timing)
def test_create_request_tracker(self):
self._test_db_create_func(models.RequestTracker,
db.create_request_tracker)
def test_create_instance_usage(self):
self._test_db_create_func(models.InstanceUsage,
db.create_instance_usage)
def test_create_instance_delete(self):
self._test_db_create_func(models.InstanceDeletes,
db.create_instance_delete)
def test_create_instance_exists(self):
self._test_db_create_func(models.InstanceExists,
db.create_instance_exists)
def _test_db_find_func(self, Model, func, select_related=True):
params = {'field1': 'value1', 'field2': 'value2'}
results = self.mox.CreateMockAnything()
if select_related:
Model.objects.select_related().AndReturn(results)
results.filter(**params).AndReturn(results)
else:
Model.objects.filter(**params).AndReturn(results)
self.mox.ReplayAll()
returned = func(**params)
self.assertEqual(returned, results)
self.mox.VerifyAll()
def test_find_lifecycles(self):
self._test_db_find_func(models.Lifecycle, db.find_lifecycles)
def test_find_timings(self):
self._test_db_find_func(models.Timing, db.find_timings)
def test_find_request_trackers(self):
self._test_db_find_func(models.RequestTracker,
db.find_request_trackers,
select_related=False)
def _test_db_get_or_create_func(self, Model, func):
params = {'field1': 'value1', 'field2': 'value2'}
object = self.mox.CreateMockAnything()
Model.objects.get_or_create(**params).AndReturn(object)
self.mox.ReplayAll()
returned = func(**params)
self.assertEqual(returned, object)
self.mox.VerifyAll()
def test_get_or_create_instance_usage(self):
self._test_db_get_or_create_func(models.InstanceUsage,
db.get_or_create_instance_usage)
def test_get_or_create_instance_delete(self):
self._test_db_get_or_create_func(models.InstanceDeletes,
db.get_or_create_instance_delete)
def test_get_instance_usage(self):
filters = {'field1': 'value1', 'field2': 'value2'}
results = self.mox.CreateMockAnything()
models.InstanceUsage.objects.filter(**filters).AndReturn(results)
results.count().AndReturn(1)
usage = self.mox.CreateMockAnything()
results[0].AndReturn(usage)
self.mox.ReplayAll()
returned = db.get_instance_usage(**filters)
self.assertEqual(returned, usage)
self.mox.VerifyAll()
def test_get_instance_delete(self):
filters = {'field1': 'value1', 'field2': 'value2'}
results = self.mox.CreateMockAnything()
models.InstanceDeletes.objects.filter(**filters).AndReturn(results)
results.count().AndReturn(1)
usage = self.mox.CreateMockAnything()
results[0].AndReturn(usage)
self.mox.ReplayAll()
returned = db.get_instance_delete(**filters)
self.assertEqual(returned, usage)
self.mox.VerifyAll()
def test_save(self):
o = self.mox.CreateMockAnything()
o.save()
self.mox.ReplayAll()
db.save(o)
self.mox.VerifyAll()

3
tests/unit/utils.py

@ -90,6 +90,7 @@ def create_raw(mox, when, event, instance=INSTANCE_ID_1,
raw.json = json_str
return raw
def create_lifecycle(mox, instance, last_state, last_task_state, last_raw):
lifecycle = mox.CreateMockAnything()
lifecycle.instance = instance
@ -98,6 +99,7 @@ def create_lifecycle(mox, instance, last_state, last_task_state, last_raw):
lifecycle.last_raw = last_raw
return lifecycle
def create_timing(mox, name, lifecycle, start_raw=None, start_when=None,
end_raw=None, end_when=None, diff=None):
timing = mox.CreateMockAnything()
@ -110,6 +112,7 @@ def create_timing(mox, name, lifecycle, start_raw=None, start_when=None,
timing.diff = diff
return timing
def create_tracker(mox, request_id, lifecycle, start, last_timing=None,
duration=str(0.0)):
tracker = mox.CreateMockAnything()

14
verifier/dbverifier.py

@ -38,12 +38,7 @@ POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
sys.path.insert(0, POSSIBLE_TOPDIR)
from stacktach import models
from stacktach import datetime_to_decimal as dt
from verifier import AmbiguousResults
from verifier import FieldMismatch
from verifier import NotFound
from verifier import VerificationException
from stacktach import logging as stacklog
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)
@ -53,7 +48,14 @@ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(messag
handler.setFormatter(formatter)
LOG.addHandler(handler)
LOG.handlers[0].doRollover()
stacklog.set_logger(LOG)
from stacktach import models
from stacktach import datetime_to_decimal as dt
from verifier import AmbiguousResults
from verifier import FieldMismatch
from verifier import NotFound
from verifier import VerificationException
def _list_exists(ending_max=None, status=None):
params = {}

7
worker/worker.py

@ -34,8 +34,7 @@ except ImportError:
from pympler.process import ProcessMemoryInfo
from stacktach import db, views
from stacktach import logging as stacklog
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)
@ -45,6 +44,10 @@ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(messag
handler.setFormatter(formatter)
LOG.addHandler(handler)
LOG.handlers[0].doRollover()
stacklog.set_logger(LOG)
from stacktach import db
from stacktach import views
class NovaConsumer(kombu.mixins.ConsumerMixin):