x/stacktach
Code Issues Proposed changes

Adding in some tests, renaming aggregate functions

Browse Source
This commit is contained in:
Andrew Melton 2013-01-22 15:38:22 -05:00
parent 1708f6a501
commit 56b37ea2df
6 changed files with 403 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
.idea/
*.pyc
local_settings.py

8
settings.py
View File

@ -4,12 +4,16 @@ import os
try:
from local_settings import *
db_engine = STACKTACH_DB_ENGINE
db_name = STACKTACH_DB_NAME
db_host = STACKTACH_DB_HOST
db_username = STACKTACH_DB_USERNAME
db_password = STACKTACH_DB_PASSWORD
install_dir = STACKTACH_INSTALL_DIR
except ImportError:
db_engine = os.environ.get('STACKTACH_DB_ENGINE',
'django.db.backends.mysql')
db_host = os.environ.get('STACKTACH_DB_HOST', "")
db_name = os.environ['STACKTACH_DB_NAME']
db_host = os.environ.get('STACKTACH_DB_HOST', "")
db_username = os.environ['STACKTACH_DB_USERNAME']
@ -27,7 +31,7 @@ MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'ENGINE': db_engine,
'NAME': db_name,
'USER': db_username,
'PASSWORD': db_password,
@ -88,7 +92,7 @@ STATICFILES_DIRS = (
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
['/root/stacktach/static/',]
[install_dir + 'static/',]
)
# List of finder classes that know how to find static files in

10
stacktach/datetime_to_decimal.py
View File

@ -17,13 +17,3 @@ def dt_from_decimal(dec):
daittyme = datetime.datetime.utcfromtimestamp(integer)
return daittyme.replace(microsecond=micro)
if __name__ == '__main__':
now = datetime.datetime.utcnow()
d = dt_to_decimal(now)
daittyme = dt_from_decimal(d)
print repr(now)
print repr(d)
print repr(daittyme)
assert(now == daittyme)

2
stacktach/models.py
View File

@ -54,7 +54,7 @@ class RawData(models.Model):
blank=True, db_index=True)
def __repr__(self):
return self.event
return "%s %s %s" % (self.event, self.instance, self.state)
class Lifecycle(models.Model):

381
stacktach/tests.py Normal file
View File

@ -0,0 +1,381 @@
import datetime
import decimal
import time
from django.utils import unittest
import datetime_to_decimal
from models import *
import views
INSTANCE_ID_1 = 'testinstanceid1'
INSTANCE_ID_2 = 'testinstanceid2'
REQUEST_ID_1 = 'testrequestid1'
REQUEST_ID_2 = 'testrequestid2'
class DatetimeToDecimalTestCase(unittest.TestCase):
def test_datetime_to_and_from_decimal(self):
now = datetime.datetime.utcnow()
d = datetime_to_decimal.dt_to_decimal(now)
daittyme = datetime_to_decimal.dt_from_decimal(d)
self.assertEqual(now, daittyme)
def test_datetime_to_decimal(self):
expected_decimal = decimal.Decimal('1356093296.123')
utc_datetime = datetime.datetime.utcfromtimestamp(expected_decimal)
actual_decimal = datetime_to_decimal.dt_to_decimal(utc_datetime)
self.assertEqual(actual_decimal, expected_decimal)
def test_decimal_to_datetime(self):
expected_decimal = decimal.Decimal('1356093296.123')
expected_datetime = datetime.datetime.utcfromtimestamp(expected_decimal)
actual_datetime = datetime_to_decimal.dt_from_decimal(expected_decimal)
self.assertEqual(actual_datetime, expected_datetime)
class ViewsUtilsTestCase(unittest.TestCase):
def test_srt_time_to_unix(self):
unix = views.str_time_to_unix('2012-12-21 12:34:56.123')
self.assertEqual(unix, decimal.Decimal('1356093296.123'))
class ViewsLifecycleTestCase(unittest.TestCase):
def setUp(self):
self.deployment = Deployment(name='TestDeployment')
self.deployment.save()
when1 = views.str_time_to_unix('2012-12-21 12:34:50.123')
when2 = views.str_time_to_unix('2012-12-21 12:34:56.123')
when3 = views.str_time_to_unix('2012-12-21 12:36:56.124')
self.update_raw = self.create_raw(self.deployment, when1,
'compute.instance.update',
host='api')
self.start_raw = self.create_raw(self.deployment, when2,
'compute.instance.reboot.start')
self.end_raw = self.create_raw(self.deployment, when3,
'compute.instance.reboot.end',
old_task='reboot')
def create_raw(self, deployment, when, event, instance=INSTANCE_ID_1,
request_id=REQUEST_ID_1, state='active', old_task='',
host='compute'):
raw_values = {
'deployment': deployment,
'host': host,
'state': state,
'old_task': old_task,
'when': when,
'event': event,
'instance': instance,
'request_id': request_id,
}
raw = RawData(**raw_values)
raw.save()
return raw
def tearDown(self):
Deployment.objects.all().delete()
RawData.objects.all().delete()
Lifecycle.objects.all().delete()
Timing.objects.all().delete()
RequestTracker.objects.all().delete()
def assertOnLifecycle(self, lifecycle, instance, last_raw):
self.assertEqual(lifecycle.instance, instance)
self.assertEqual(lifecycle.last_raw.id, last_raw.id)
self.assertEqual(lifecycle.last_state, last_raw.state)
self.assertEqual(lifecycle.last_task_state, last_raw.old_task)
def assertOnTiming(self, timing, lifecycle, start_raw, end_raw, diff):
self.assertEqual(timing.lifecycle.id, lifecycle.id)
self.assertEqual(timing.start_raw.id, start_raw.id)
self.assertEqual(timing.end_raw.id, end_raw.id)
self.assertEqual(timing.start_when, start_raw.when)
self.assertEqual(timing.end_when, end_raw.when)
self.assertEqual(timing.diff, decimal.Decimal(diff))
def assertOnTracker(self, tracker, request_id, lifecycle, start, diff=None):
self.assertEqual(tracker.request_id, request_id)
self.assertEqual(tracker.lifecycle.id, lifecycle.id)
self.assertEqual(tracker.start, start)
if diff:
self.assertEqual(tracker.duration, diff)
def test_aggregate_lifecycle_and_timing(self):
views.aggregate_lifecycle(self.update_raw)
views.aggregate_lifecycle(self.start_raw)
lifecycles = Lifecycle.objects.select_related()\
.filter(instance=INSTANCE_ID_1)
self.assertEqual(len(lifecycles), 1)
lifecycle = lifecycles[0]
self.assertOnLifecycle(lifecycle, INSTANCE_ID_1, self.start_raw)
views.aggregate_lifecycle(self.end_raw)
lifecycles = Lifecycle.objects.select_related()\
.filter(instance=INSTANCE_ID_1)
self.assertEqual(len(lifecycles), 1)
lifecycle = lifecycles[0]
self.assertOnLifecycle(lifecycle, INSTANCE_ID_1, self.end_raw)
timings = Timing.objects.select_related()\
.filter(lifecycle=lifecycle)
self.assertEqual(len(lifecycles), 1)
timing = timings[0]
expected_diff = self.end_raw.when - self.start_raw.when
self.assertOnTiming(timing, lifecycle, self.start_raw, self.end_raw,
expected_diff)
def test_multiple_instance_lifecycles(self):
when1 = views.str_time_to_unix('2012-12-21 13:32:50.123')
when2 = views.str_time_to_unix('2012-12-21 13:34:50.123')
when3 = views.str_time_to_unix('2012-12-21 13:37:50.124')
update_raw2 = self.create_raw(self.deployment, when1,
'compute.instance.update',
instance=INSTANCE_ID_2,
request_id=REQUEST_ID_2,
host='api')
start_raw2 = self.create_raw(self.deployment, when2,
'compute.instance.resize.start',
instance=INSTANCE_ID_2,
request_id=REQUEST_ID_2)
end_raw2 = self.create_raw(self.deployment, when3,
'compute.instance.resize.end',
old_task='resize',
instance=INSTANCE_ID_2,
request_id=REQUEST_ID_2)
views.aggregate_lifecycle(self.update_raw)
views.aggregate_lifecycle(self.start_raw)
views.aggregate_lifecycle(update_raw2)
views.aggregate_lifecycle(start_raw2)
lifecycles = Lifecycle.objects.all().order_by('id')
self.assertEqual(len(lifecycles), 2)
lifecycle1 = lifecycles[0]
self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, self.start_raw)
lifecycle2 = lifecycles[1]
self.assertOnLifecycle(lifecycle2, INSTANCE_ID_2, start_raw2)
views.aggregate_lifecycle(end_raw2)
views.aggregate_lifecycle(self.end_raw)
lifecycles = Lifecycle.objects.all().order_by('id')
self.assertEqual(len(lifecycles), 2)
lifecycle1 = lifecycles[0]
self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, self.end_raw)
lifecycle2 = lifecycles[1]
self.assertOnLifecycle(lifecycle2, INSTANCE_ID_2, end_raw2)
timings = Timing.objects.all().order_by('id')
self.assertEqual(len(timings), 2)
timing1 = timings[0]
expected_diff1 = self.end_raw.when - self.start_raw.when
self.assertOnTiming(timing1, lifecycle1, self.start_raw, self.end_raw,
expected_diff1)
expected_diff2 = end_raw2.when - start_raw2.when
timing2 = timings[1]
self.assertOnTiming(timing2, lifecycle2, start_raw2, end_raw2,
expected_diff2)
def test_same_instance_multiple_timings(self):
when1 = views.str_time_to_unix('2012-12-21 13:32:50.123')
when2 = views.str_time_to_unix('2012-12-21 13:34:50.123')
when3 = views.str_time_to_unix('2012-12-21 13:37:50.124')
update_raw2 = self.create_raw(self.deployment, when1,
'compute.instance.update',
request_id=REQUEST_ID_2,
host='api')
start_raw2 = self.create_raw(self.deployment, when2,
'compute.instance.resize.start',
request_id=REQUEST_ID_2)
end_raw2 = self.create_raw(self.deployment, when3,
'compute.instance.resize.end',
old_task='resize',
request_id=REQUEST_ID_2)
# First action started
views.aggregate_lifecycle(self.update_raw)
views.aggregate_lifecycle(self.start_raw)
# Second action started, first end is late
views.aggregate_lifecycle(update_raw2)
views.aggregate_lifecycle(start_raw2)
# Finally get first end
views.aggregate_lifecycle(self.end_raw)
# Second end
views.aggregate_lifecycle(end_raw2)
lifecycles = Lifecycle.objects.select_related()\
.filter(instance=INSTANCE_ID_1)
self.assertEqual(len(lifecycles), 1)
lifecycle1 = lifecycles[0]
self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, end_raw2)
timings = Timing.objects.all().order_by('id')
self.assertEqual(len(timings), 2)
timing1 = timings[0]
expected_diff1 = self.end_raw.when - self.start_raw.when
self.assertOnTiming(timing1, lifecycle1, self.start_raw, self.end_raw,
expected_diff1)
expected_diff2 = end_raw2.when - start_raw2.when
timing2 = timings[1]
self.assertOnTiming(timing2, lifecycle1, start_raw2, end_raw2,
expected_diff2)
def test_aggregate_lifecycle_and_kpi(self):
views.aggregate_lifecycle(self.update_raw)
lifecycles = Lifecycle.objects.select_related()\
.filter(instance=INSTANCE_ID_1)
self.assertEqual(len(lifecycles), 1)
lifecycle = lifecycles[0]
self.assertOnLifecycle(lifecycle, INSTANCE_ID_1, self.update_raw)
trackers = RequestTracker.objects.filter(request_id=REQUEST_ID_1)
self.assertEqual(len(trackers), 1)
tracker = trackers[0]
self.assertOnTracker(tracker, REQUEST_ID_1, lifecycle,
self.update_raw.when)
views.aggregate_lifecycle(self.start_raw)
views.aggregate_lifecycle(self.end_raw)
trackers = RequestTracker.objects.filter(request_id=REQUEST_ID_1)
self.assertEqual(len(trackers), 1)
tracker = trackers[0]
expected_diff = self.end_raw.when-self.update_raw.when
self.assertOnTracker(tracker, REQUEST_ID_1, lifecycle,
self.update_raw.when, expected_diff)
def test_multiple_instance_kpi(self):
when1 = views.str_time_to_unix('2012-12-21 13:32:50.123')
when2 = views.str_time_to_unix('2012-12-21 13:34:50.123')
when3 = views.str_time_to_unix('2012-12-21 13:37:50.124')
update_raw2 = self.create_raw(self.deployment, when1,
'compute.instance.update',
instance=INSTANCE_ID_2,
request_id=REQUEST_ID_2,
host='api')
start_raw2 = self.create_raw(self.deployment, when2,
'compute.instance.resize.start',
instance=INSTANCE_ID_2,
request_id=REQUEST_ID_2)
end_raw2 = self.create_raw(self.deployment, when3,
'compute.instance.resize.end',
instance=INSTANCE_ID_2,
old_task='resize',
request_id=REQUEST_ID_2)
views.aggregate_lifecycle(self.update_raw)
views.aggregate_lifecycle(self.start_raw)
views.aggregate_lifecycle(self.end_raw)
views.aggregate_lifecycle(update_raw2)
views.aggregate_lifecycle(start_raw2)
views.aggregate_lifecycle(end_raw2)
lifecycles = Lifecycle.objects.all().order_by('id')
self.assertEqual(len(lifecycles), 2)
lifecycle1 = lifecycles[0]
self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, self.end_raw)
lifecycle2 = lifecycles[1]
self.assertOnLifecycle(lifecycle2, INSTANCE_ID_2, end_raw2)
trackers = RequestTracker.objects.all().order_by('id')
self.assertEqual(len(trackers), 2)
tracker1 = trackers[0]
expected_diff = self.end_raw.when-self.update_raw.when
self.assertOnTracker(tracker1, REQUEST_ID_1, lifecycle1,
self.update_raw.when, expected_diff)
tracker2 = trackers[1]
expected_diff2 = end_raw2.when-update_raw2.when
self.assertOnTracker(tracker2, REQUEST_ID_2, lifecycle2,
update_raw2.when, expected_diff2)
def test_single_instance_multiple_kpi(self):
when1 = views.str_time_to_unix('2012-12-21 13:32:50.123')
when2 = views.str_time_to_unix('2012-12-21 13:34:50.123')
when3 = views.str_time_to_unix('2012-12-21 13:37:50.124')
update_raw2 = self.create_raw(self.deployment, when1,
'compute.instance.update',
request_id=REQUEST_ID_2,
host='api')
start_raw2 = self.create_raw(self.deployment, when2,
'compute.instance.resize.start',
request_id=REQUEST_ID_2)
end_raw2 = self.create_raw(self.deployment, when3,
'compute.instance.resize.end',
old_task='resize',
request_id=REQUEST_ID_2)
views.aggregate_lifecycle(self.update_raw)
views.aggregate_lifecycle(self.start_raw)
views.aggregate_lifecycle(self.end_raw)
views.aggregate_lifecycle(update_raw2)
views.aggregate_lifecycle(start_raw2)
views.aggregate_lifecycle(end_raw2)
lifecycles = Lifecycle.objects.all().order_by('id')
self.assertEqual(len(lifecycles), 1)
lifecycle1 = lifecycles[0]
self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, end_raw2)
trackers = RequestTracker.objects.all().order_by('id')
self.assertEqual(len(trackers), 2)
tracker1 = trackers[0]
expected_diff1 = self.end_raw.when-self.update_raw.when
self.assertOnTracker(tracker1, REQUEST_ID_1, lifecycle1,
self.update_raw.when, expected_diff1)
tracker2 = trackers[1]
expected_diff2 = end_raw2.when-update_raw2.when
self.assertOnTracker(tracker2, REQUEST_ID_2, lifecycle1,
update_raw2.when, expected_diff2)
def test_single_instance_multiple_kpi_out_of_order(self):
when1 = views.str_time_to_unix('2012-12-21 13:32:50.123')
when2 = views.str_time_to_unix('2012-12-21 13:34:50.123')
when3 = views.str_time_to_unix('2012-12-21 13:37:50.124')
update_raw2 = self.create_raw(self.deployment, when1,
'compute.instance.update',
request_id=REQUEST_ID_2,
host='api')
start_raw2 = self.create_raw(self.deployment, when2,
'compute.instance.resize.start',
request_id=REQUEST_ID_2)
end_raw2 = self.create_raw(self.deployment, when3,
'compute.instance.resize.end',
old_task='resize',
request_id=REQUEST_ID_2)
# First action started
views.aggregate_lifecycle(self.update_raw)
views.aggregate_lifecycle(self.start_raw)
# Second action started, first end is late
views.aggregate_lifecycle(update_raw2)
views.aggregate_lifecycle(start_raw2)
# Finally get first end
views.aggregate_lifecycle(self.end_raw)
# Second end
views.aggregate_lifecycle(end_raw2)
lifecycles = Lifecycle.objects.all().order_by('id')
self.assertEqual(len(lifecycles), 1)
lifecycle1 = lifecycles[0]
self.assertOnLifecycle(lifecycle1, INSTANCE_ID_1, end_raw2)
trackers = RequestTracker.objects.all().order_by('id')
self.assertEqual(len(trackers), 2)
tracker1 = trackers[0]
expected_diff1 = self.end_raw.when-self.update_raw.when
self.assertOnTracker(tracker1, REQUEST_ID_1, lifecycle1,
self.update_raw.when, expected_diff1)
tracker2 = trackers[1]
expected_diff2 = end_raw2.when-update_raw2.when
self.assertOnTracker(tracker2, REQUEST_ID_2, lifecycle1,
update_raw2.when, expected_diff2)

22
stacktach/views.py
View File

@ -113,7 +113,7 @@ def update_kpi(lifecycle, timing, raw):
tracker.save()
def aggregate(raw):
def aggregate_lifecycle(raw):
"""Roll up the raw event into a Lifecycle object
and a bunch of Timing objects.
@ -211,10 +211,10 @@ INSTANCE_EVENT = {
'exists': 'compute.instance.exists',
}
def process_for_usage(raw):
def aggregate_usage(raw):
if not raw.instance:
return
if raw.event == INSTANCE_EVENT['create_start'] or \
raw.event == INSTANCE_EVENT['resize_prep_start'] or\
raw.event == INSTANCE_EVENT['resize_revert_start']:
@ -239,10 +239,10 @@ def _process_usage_for_new_launch(raw):
values = {}
values['instance'] = payload['instance_id']
values['request_id'] = notif[1]['_context_request_id']
if raw.event == INSTANCE_EVENT['create_start']:
values['instance_type_id'] = payload['instance_type_id']
usage = models.InstanceUsage(**values)
usage.save()
@ -259,10 +259,10 @@ def _process_usage_for_updates(raw):
raw.event == INSTANCE_EVENT['resize_finish_end'] or\
raw.event == INSTANCE_EVENT['resize_revert_end']:
instance.launched_at = str_time_to_unix(payload['launched_at'])
if raw.event == INSTANCE_EVENT['resize_revert_end']:
instance.instance_type_id = payload['instance_type_id']
elif raw.event == INSTANCE_EVENT['resize_prep_end']:
elif raw.event == INSTANCE_EVENT['resize_prep_end']:
instance.instance_type_id = payload['new_instance_type_id']
instance.save()
@ -293,14 +293,14 @@ def _process_exists(raw):
values['instance'] = instance_id
values['launched_at'] = launched_at
values['instance_type_id'] = payload['instance_type_id']
values['usage'] = usage
values['raw'] = raw
deleted_at = payload.get('deleted_at')
if deleted_at and deleted_at != '':
deleted_at = str_time_to_unix(deleted_at)
values['deleted_at'] = deleted_at
exists = models.InstanceExists(**values)
exists.save()
@ -348,8 +348,8 @@ def process_raw_data(deployment, args, json_args):
record = models.RawData(**values)
record.save()
aggregate(record)
process_for_usage(record)
aggregate_lifecycle(record)
aggregate_usage(record)
return record