x/distil
Code Issues Proposed changes

Removed GaugeAverage since it is unlikely to be needed

Browse Source 
and easy to reimplement later.

added tests for GaugeMax and CumulativeRange

CumulativeTotal was renamed to CumulativeRange
This commit is contained in:
adriant 2014-03-21 15:04:21 +13:00
parent 2b4a3d080c
commit 8686f28659
3 changed files with 174 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
artifice/models/resources.py
View File

@ -121,6 +121,6 @@ class Volume(BaseModelConstruct):
class Network(BaseModelConstruct): class Network(BaseModelConstruct):
relevant_meters = ["network.outgoing.bytes", "network.incoming.bytes"] relevant_meters = ["network.outgoing.bytes", "network.incoming.bytes"]
transformer = transformers.CumulativeTotal() transformer = transformers.CumulativeRange()
type = "network_interface" type = "network_interface"

24
artifice/transformers.py
View File

@ -3,6 +3,10 @@ import constants
import helpers import helpers
class TransformerValidationError(BaseException):
pass
class Transformer(object): class Transformer(object):
meter_type = None meter_type = None
@ -16,13 +20,15 @@ class Transformer(object):
if self.meter_type is None: if self.meter_type is None:
for meter in self.required_meters: for meter in self.required_meters:
if meter not in meters: if meter not in meters:
raise AttributeError("Required meters: " + raise TransformerValidationError(
str(self.required_meters)) "Required meters: " +
str(self.required_meters))
else: else:
for meter in meters.values(): for meter in meters.values():
if meter.type != self.meter_type: if meter.type != self.meter_type:
raise AttributeError("Meters must all be of type: " + raise TransformerValidationError(
self.meter_type) "Meters must all be of type: " +
self.meter_type)
def _transform_usage(self, meters): def _transform_usage(self, meters):
raise NotImplementedError raise NotImplementedError
@ -101,19 +107,19 @@ class GaugeMax(Transformer):
def _transform_usage(self, meters): def _transform_usage(self, meters):
usage_dict = {} usage_dict = {}
for meter in meters.values(): for name, meter in meters.iteritems():
usage = meter.usage() usage = meter.usage()
max_vol = max([v["counter_volume"] for v in usage]) max_vol = max([v["counter_volume"] for v in usage])
usage_dict[meter.name] = max_vol usage_dict[name] = max_vol
return usage_dict return usage_dict
class CumulativeTotal(Transformer): class CumulativeRange(Transformer):
meter_type = 'cumulative' meter_type = 'cumulative'
def _transform_usage(self, meters): def _transform_usage(self, meters):
usage_dict = {} usage_dict = {}
for meter in meters.values(): for name, meter in meters.iteritems():
measurements = meter.usage() measurements = meter.usage()
measurements = sorted(measurements, key=lambda x: x["timestamp"]) measurements = sorted(measurements, key=lambda x: x["timestamp"])
count = 0 count = 0
@ -131,5 +137,5 @@ class CumulativeTotal(Transformer):
if count > 1: if count > 1:
total_usage = usage - measurements[0]["counter_volume"] total_usage = usage - measurements[0]["counter_volume"]
usage_dict[meter.name] = total_usage usage_dict[name] = total_usage
return usage_dict return usage_dict

179
tests/test_transformers.py
View File

@ -1,22 +1,32 @@
import artifice.transformers import artifice.transformers
import artifice.constants from artifice.transformers import TransformerValidationError
import artifice.constants as constants
import unittest import unittest
import mock import mock
class testdata: class testdata:
t0 = '2014-01-01T00:00:00' t0 = '2014-01-01T00:00:00'
t1 = '2014-01-01T01:00:00' t0_10 = '2014-01-01T00:10:00'
flavor = 'm1.tiny' t0_20 = '2014-01-01T00:30:00'
t0_30 = '2014-01-01T00:30:00' t0_30 = '2014-01-01T00:30:00'
t0_40 = '2014-01-01T00:40:00'
t0_50 = '2014-01-01T00:50:00'
t1 = '2014-01-01T01:00:00'
flavor = 'm1.tiny'
flavor2 = 'm1.large' flavor2 = 'm1.large'
class TestMeter(object): class TestMeter(object):
def __init__(self, data): def __init__(self, data, mtype=None):
self.data = data self.data = data
self.type = mtype
def usage(self): def usage(self):
return self.data return self.data
class UptimeTransformerTests(unittest.TestCase): class UptimeTransformerTests(unittest.TestCase):
def test_required_metrics_not_present(self): def test_required_metrics_not_present(self):
""" """
@ -24,8 +34,8 @@ class UptimeTransformerTests(unittest.TestCase):
is not present. is not present.
""" """
xform = artifice.transformers.Uptime() xform = artifice.transformers.Uptime()
with self.assertRaises(AttributeError) as e: with self.assertRaises(TransformerValidationError) as e:
xform.transform_usage({}) xform.transform_usage({})
self.assertTrue(e.exception.message.startswith('Required meters:')) self.assertTrue(e.exception.message.startswith('Required meters:'))
@ -59,8 +69,8 @@ class UptimeTransformerTests(unittest.TestCase):
{'timestamp': testdata.t1, 'counter_volume': testdata.flavor}, {'timestamp': testdata.t1, 'counter_volume': testdata.flavor},
]), ]),
'state': TestMeter([ 'state': TestMeter([
{'timestamp': testdata.t0, 'counter_volume': artifice.constants.active}, {'timestamp': testdata.t0, 'counter_volume': constants.active},
{'timestamp': testdata.t1, 'counter_volume': artifice.constants.active} {'timestamp': testdata.t1, 'counter_volume': constants.active}
]), ]),
} }
@ -73,7 +83,6 @@ class UptimeTransformerTests(unittest.TestCase):
Test that a machine offline for a 1h period with constant flavor Test that a machine offline for a 1h period with constant flavor
works and gives zero uptime. works and gives zero uptime.
""" """
xform = artifice.transformers.Uptime()
meters = { meters = {
'flavor': TestMeter([ 'flavor': TestMeter([
@ -81,8 +90,8 @@ class UptimeTransformerTests(unittest.TestCase):
{'timestamp': testdata.t1, 'counter_volume': testdata.flavor}, {'timestamp': testdata.t1, 'counter_volume': testdata.flavor},
]), ]),
'state': TestMeter([ 'state': TestMeter([
{'timestamp': testdata.t0, 'counter_volume': artifice.constants.stopped}, {'timestamp': testdata.t0, 'counter_volume': constants.stopped},
{'timestamp': testdata.t1, 'counter_volume': artifice.constants.stopped} {'timestamp': testdata.t1, 'counter_volume': constants.stopped}
]), ]),
} }
@ -90,7 +99,6 @@ class UptimeTransformerTests(unittest.TestCase):
# there should be no usage, the machine was off. # there should be no usage, the machine was off.
self.assertEqual({}, result) self.assertEqual({}, result)
def test_shutdown_during_period(self): def test_shutdown_during_period(self):
""" """
Test that a machine run for 0.5 then shutdown gives 0.5h uptime. Test that a machine run for 0.5 then shutdown gives 0.5h uptime.
@ -102,9 +110,9 @@ class UptimeTransformerTests(unittest.TestCase):
{'timestamp': testdata.t1, 'counter_volume': testdata.flavor}, {'timestamp': testdata.t1, 'counter_volume': testdata.flavor},
]), ]),
'state': TestMeter([ 'state': TestMeter([
{'timestamp': testdata.t0, 'counter_volume': artifice.constants.active}, {'timestamp': testdata.t0, 'counter_volume': constants.active},
{'timestamp': testdata.t0_30, 'counter_volume': artifice.constants.stopped}, {'timestamp': testdata.t0_30, 'counter_volume': constants.stopped},
{'timestamp': testdata.t1, 'counter_volume': artifice.constants.stopped} {'timestamp': testdata.t1, 'counter_volume': constants.stopped}
]), ]),
} }
@ -114,8 +122,8 @@ class UptimeTransformerTests(unittest.TestCase):
def test_online_flavor_change(self): def test_online_flavor_change(self):
""" """
Test that a machine run for 0.5h as m1.tiny, resized to m1.large, and run Test that a machine run for 0.5h as m1.tiny, resized to m1.large,
for a further 0.5 yields 0.5h of uptime in each class. and run for a further 0.5 yields 0.5h of uptime in each class.
""" """
meters = { meters = {
'flavor': TestMeter([ 'flavor': TestMeter([
@ -124,12 +132,141 @@ class UptimeTransformerTests(unittest.TestCase):
{'timestamp': testdata.t1, 'counter_volume': testdata.flavor2}, {'timestamp': testdata.t1, 'counter_volume': testdata.flavor2},
]), ]),
'state': TestMeter([ 'state': TestMeter([
{'timestamp': testdata.t0, 'counter_volume': artifice.constants.active}, {'timestamp': testdata.t0, 'counter_volume': constants.active},
{'timestamp': testdata.t0_30, 'counter_volume': artifice.constants.active}, {'timestamp': testdata.t0_30, 'counter_volume': constants.active},
{'timestamp': testdata.t1, 'counter_volume': artifice.constants.active} {'timestamp': testdata.t1, 'counter_volume': constants.active}
]), ]),
} }
result = self._run_transform(meters) result = self._run_transform(meters)
# there should be half an hour of usage in each of m1.tiny and m1.large # there should be half an hour of usage in each of m1.tiny and m1.large
self.assertEqual({'m1.tiny': 1800, 'm1.large': 1800}, result) self.assertEqual({'m1.tiny': 1800, 'm1.large': 1800}, result)
class GaugeMaxTransformerTests(unittest.TestCase):
def test_wrong_metrics_type(self):
"""
Test that the correct exception is thrown if any given meters
are of the wrong type.
"""
xform = artifice.transformers.GaugeMax()
meter = mock.MagicMock()
meter.type = "cumulative"
with self.assertRaises(TransformerValidationError) as e:
xform.transform_usage({'some_meter': meter})
self.assertTrue(
e.exception.message.startswith('Meters must all be of type: '))
def test_all_different_values(self):
"""
Tests that the transformer correctly grabs the highest value,
when all values are different.
"""
meters = {
'size': TestMeter([
{'timestamp': testdata.t0, 'counter_volume': 12},
{'timestamp': testdata.t0_10, 'counter_volume': 3},
{'timestamp': testdata.t0_20, 'counter_volume': 7},
{'timestamp': testdata.t0_30, 'counter_volume': 3},
{'timestamp': testdata.t0_40, 'counter_volume': 25},
{'timestamp': testdata.t0_50, 'counter_volume': 2},
{'timestamp': testdata.t1, 'counter_volume': 6},
], "gauge")
}
xform = artifice.transformers.GaugeMax()
usage = xform.transform_usage(meters)
self.assertEqual({'size': 25}, usage)
def test_all_same_values(self):
"""
Tests that that transformer correctly grabs any value,
when all values are the same.
"""
meters = {
'size': TestMeter([
{'timestamp': testdata.t0, 'counter_volume': 25},
{'timestamp': testdata.t0_30, 'counter_volume': 25},
{'timestamp': testdata.t1, 'counter_volume': 25},
], "gauge")
}
xform = artifice.transformers.GaugeMax()
usage = xform.transform_usage(meters)
self.assertEqual({'size': 25}, usage)
class CumulativeRangeTransformerTests(unittest.TestCase):
def test_no_reset(self):
"""
Tests that the correct usage is being returned for the range,
when no reset occurs.
"""
meters = {
'time': TestMeter([
{'timestamp': testdata.t0, 'counter_volume': 1},
{'timestamp': testdata.t0_10, 'counter_volume': 2},
{'timestamp': testdata.t0_20, 'counter_volume': 3},
{'timestamp': testdata.t0_30, 'counter_volume': 4},
{'timestamp': testdata.t0_40, 'counter_volume': 5},
{'timestamp': testdata.t0_50, 'counter_volume': 6},
{'timestamp': testdata.t1, 'counter_volume': 7},
], "cumulative")
}
xform = artifice.transformers.CumulativeRange()
usage = xform.transform_usage(meters)
self.assertEqual({'time': 6}, usage)
def test_clear_reset(self):
"""
Tests that the correct usage is being returned for the range,
when a reset occurs.
"""
meters = {
'time': TestMeter([
{'timestamp': testdata.t0, 'counter_volume': 10},
{'timestamp': testdata.t0_10, 'counter_volume': 20},
{'timestamp': testdata.t0_20, 'counter_volume': 40},
{'timestamp': testdata.t0_30, 'counter_volume': 0},
{'timestamp': testdata.t0_40, 'counter_volume': 20},
{'timestamp': testdata.t0_50, 'counter_volume': 30},
{'timestamp': testdata.t1, 'counter_volume': 40},
], "cumulative")
}
xform = artifice.transformers.CumulativeRange()
usage = xform.transform_usage(meters)
self.assertEqual({'time': 70}, usage)
def test_close_reset(self):
"""
Tests that the correct usage is being returned for the range,
when a very close reset occurs.
"""
meters = {
'time': TestMeter([
{'timestamp': testdata.t0, 'counter_volume': 10},
{'timestamp': testdata.t0_10, 'counter_volume': 20},
{'timestamp': testdata.t0_20, 'counter_volume': 40},
{'timestamp': testdata.t0_30, 'counter_volume': 39},
{'timestamp': testdata.t0_40, 'counter_volume': 50},
{'timestamp': testdata.t0_50, 'counter_volume': 60},
{'timestamp': testdata.t1, 'counter_volume': 70},
], "cumulative")
}
xform = artifice.transformers.CumulativeRange()
usage = xform.transform_usage(meters)
self.assertEqual({'time': 100}, usage)