First implementation of transformers. Seems to work, all old tests pass.
Uptime doesn't use flavor metric yet. Transformers need testing.
This commit is contained in:
adriant
parent
0cf0680220
commit
9188c7a6cf
@ -1,154 +0,0 @@
|
||||
import datetime
|
||||
from constants import date_format, other_date_format
|
||||
|
||||
|
||||
class Artifact(object):
|
||||
|
||||
"""
|
||||
Provides base artifact controls; generic typing information
|
||||
for the artifact structures.
|
||||
"""
|
||||
|
||||
def __init__(self, resource, usage, start, end):
|
||||
|
||||
self.resource = resource
|
||||
self.usage = usage # Raw meter data from Ceilometer
|
||||
self.start = start
|
||||
self.end = end
|
||||
|
||||
def __getitem__(self, item):
|
||||
if item in self._data:
|
||||
return self._data[item]
|
||||
raise KeyError("no such item %s" % item)
|
||||
|
||||
def volume(self):
|
||||
"""
|
||||
Default billable number for this volume
|
||||
"""
|
||||
return sum([x["counter_volume"] for x in self.usage])
|
||||
|
||||
|
||||
class Cumulative(Artifact):
|
||||
|
||||
def volume(self):
|
||||
measurements = self.usage
|
||||
measurements = sorted(measurements, key=lambda x: x["timestamp"])
|
||||
count = 0
|
||||
usage = 0
|
||||
last_measure = None
|
||||
for measure in measurements:
|
||||
if last_measure is not None and (measure["counter_volume"] <
|
||||
last_measure["counter_volume"]):
|
||||
usage = usage + last_measure["counter_volume"]
|
||||
count = count + 1
|
||||
last_measure = measure
|
||||
|
||||
usage = usage + measurements[-1]["counter_volume"]
|
||||
|
||||
if count > 1:
|
||||
total_usage = usage - measurements[0]["counter_volume"]
|
||||
return total_usage
|
||||
|
||||
|
||||
# Gauge and Delta have very little to do: They are expected only to
|
||||
# exist as "not a cumulative" sort of artifact.
|
||||
class Gauge(Artifact):
|
||||
|
||||
def volume(self):
|
||||
"""
|
||||
Default billable number for this volume
|
||||
"""
|
||||
# print "Usage is %s" % self.usage
|
||||
usage = sorted(self.usage, key=lambda x: x["timestamp"])
|
||||
|
||||
blocks = []
|
||||
curr = [usage[0]]
|
||||
last = usage[0]
|
||||
try:
|
||||
last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
|
||||
date_format)
|
||||
except ValueError:
|
||||
last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
|
||||
other_date_format)
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
for val in usage[1:]:
|
||||
try:
|
||||
val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
|
||||
date_format)
|
||||
except ValueError:
|
||||
val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
|
||||
other_date_format)
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
difference = (val['timestamp'] - last["timestamp"])
|
||||
if difference > datetime.timedelta(hours=1):
|
||||
blocks.append(curr)
|
||||
curr = [val]
|
||||
last = val
|
||||
else:
|
||||
curr.append(val)
|
||||
|
||||
# this adds the last remaining values as a block of their own on exit
|
||||
# might mean people are billed twice for an hour at times...
|
||||
# but solves the issue of not billing if there isn't enough data for
|
||||
# full hour.
|
||||
blocks.append(curr)
|
||||
|
||||
# We are now sorted into 1-hour blocks
|
||||
totals = []
|
||||
for block in blocks:
|
||||
usage = max([v["counter_volume"] for v in block])
|
||||
totals.append(usage)
|
||||
|
||||
# totals = [max(x, key=lambda val: val["counter_volume"] ) for x in blocks]
|
||||
# totals is now an array of max values per hour for a given month.
|
||||
return sum(totals)
|
||||
|
||||
# This continues to be wrong.
|
||||
def uptime(self, tracked):
|
||||
"""Calculates uptime accurately for the given 'tracked' states.
|
||||
- Will ignore all other states.
|
||||
- Relies heavily on the existence of a state meter, and
|
||||
should only ever be called on the state meter.
|
||||
|
||||
Returns: uptime in seconds"""
|
||||
|
||||
usage = sorted(self.usage, key=lambda x: x["timestamp"])
|
||||
|
||||
last = usage[0]
|
||||
try:
|
||||
last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
|
||||
date_format)
|
||||
except ValueError:
|
||||
last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
|
||||
other_date_format)
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
uptime = 0.0
|
||||
|
||||
for val in usage[1:]:
|
||||
try:
|
||||
val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
|
||||
date_format)
|
||||
except ValueError:
|
||||
val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
|
||||
other_date_format)
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
if val["counter_volume"] in tracked:
|
||||
difference = val["timestamp"] - last["timestamp"]
|
||||
|
||||
uptime = uptime + difference.seconds
|
||||
|
||||
last = val
|
||||
|
||||
return uptime
|
||||
|
||||
|
||||
class Delta(Artifact):
|
||||
pass
|
||||
@ -30,13 +30,7 @@ class Database(object):
|
||||
in a resource, for all the resources given"""
|
||||
|
||||
for resource in usage:
|
||||
for key in resource.usage_strategies:
|
||||
strategy = resource.usage_strategies[key]
|
||||
volume = resource.get(strategy['usage'])
|
||||
try:
|
||||
service = resource.get(strategy['service'])
|
||||
except AttributeError:
|
||||
service = strategy['service']
|
||||
for service, volume in resource.usage().iteritems():
|
||||
resource_id = resource.get("resource_id")
|
||||
tenant_id = resource.get("tenant_id")
|
||||
|
||||
|
||||
@ -1,7 +1,5 @@
|
||||
import requests
|
||||
import json
|
||||
from collections import defaultdict
|
||||
import artifact
|
||||
import auth
|
||||
from ceilometerclient.v2.client import Client as ceilometer
|
||||
from .models import resources
|
||||
@ -275,7 +273,27 @@ class Meter(object):
|
||||
self.conn = conn
|
||||
self.start = start
|
||||
self.end = end
|
||||
# self.meter = meter
|
||||
|
||||
self.measurements = self.get_meter(start, end,
|
||||
self.conn.auth.auth_token)
|
||||
|
||||
self.type = set([a["counter_type"] for a in self.measurements])
|
||||
if len(self.type) > 1:
|
||||
# That's a big problem
|
||||
raise RuntimeError("Too many types for measurement!")
|
||||
elif len(self.type) == 0:
|
||||
raise RuntimeError("No types!")
|
||||
else:
|
||||
self.type = self.type.pop()
|
||||
|
||||
self.name = set([a["counter_name"] for a in self.measurements])
|
||||
if len(self.name) > 1:
|
||||
# That's a big problem
|
||||
raise RuntimeError("Too many names for measurement!")
|
||||
elif len(self.name) == 0:
|
||||
raise RuntimeError("No types!")
|
||||
else:
|
||||
self.name = self.name.pop()
|
||||
|
||||
def get_meter(self, start, end, auth):
|
||||
# Meter is a href; in this case, it has a set of fields with it already.
|
||||
@ -294,35 +312,5 @@ class Meter(object):
|
||||
)
|
||||
return json.loads(r.text)
|
||||
|
||||
def volume(self):
|
||||
return self.usage(self.start, self.end)
|
||||
|
||||
def usage(self, start, end):
|
||||
"""
|
||||
Usage condenses the entirety of a meter into a single datapoint:
|
||||
A volume value that we can plot as a single number against previous
|
||||
usage for a given range.
|
||||
"""
|
||||
measurements = self.get_meter(start, end, self.conn.auth.auth_token)
|
||||
# return measurements
|
||||
|
||||
# print measurements
|
||||
|
||||
# self.measurements = defaultdict(list)
|
||||
self.type = set([a["counter_type"] for a in measurements])
|
||||
if len(self.type) > 1:
|
||||
# That's a big problem
|
||||
raise RuntimeError("Too many types for measurement!")
|
||||
elif len(self.type) == 0:
|
||||
raise RuntimeError("No types!")
|
||||
else:
|
||||
self.type = self.type.pop()
|
||||
type_ = None
|
||||
if self.type == "cumulative":
|
||||
type_ = artifact.Cumulative
|
||||
elif self.type == "gauge":
|
||||
type_ = artifact.Gauge
|
||||
elif self.type == "delta":
|
||||
type_ = artifact.Delta
|
||||
|
||||
return type_(self.resource, measurements, start, end)
|
||||
def usage(self):
|
||||
return self.measurements
|
||||
|
||||
@ -1,7 +1,4 @@
|
||||
from decimal import Decimal
|
||||
import math
|
||||
|
||||
from artifice import constants
|
||||
from . import transformers
|
||||
|
||||
|
||||
class BaseModelConstruct(object):
|
||||
@ -34,37 +31,28 @@ class BaseModelConstruct(object):
|
||||
# information based on a meter, I guess?
|
||||
return getattr(self, name)
|
||||
|
||||
def usage(self):
|
||||
def meters(self):
|
||||
dct = {}
|
||||
for meter in self.relevant_meters:
|
||||
try:
|
||||
vol = self._raw.meter(meter, self.start, self.end).volume()
|
||||
dct[meter] = vol
|
||||
mtr = self._raw.meter(meter, self.start, self.end)
|
||||
dct[meter] = mtr
|
||||
except AttributeError:
|
||||
# This is OK. We're not worried about non-existent meters,
|
||||
# I think. For now, anyway.
|
||||
pass
|
||||
return dct
|
||||
|
||||
def save(self):
|
||||
for meter in self.relevant_meters:
|
||||
try:
|
||||
self._raw.meter(meter, self.start, self.end).save()
|
||||
except AttributeError:
|
||||
# This is OK. We're not worried about non-existent meters,
|
||||
# I think. For now, anyway.
|
||||
pass
|
||||
|
||||
|
||||
def to_mb(bytes):
|
||||
# function to make code easier to understand elsewhere.
|
||||
return (bytes / 1000) / 1000
|
||||
def usage(self):
|
||||
meters = self.meters()
|
||||
usage = self.transformer.transform_usage(meters)
|
||||
return usage
|
||||
|
||||
|
||||
class VM(BaseModelConstruct):
|
||||
relevant_meters = ["state"]
|
||||
|
||||
usage_strategies = {"uptime": {"usage": "uptime", "service": "flavor"}}
|
||||
transformer = transformers.Uptime()
|
||||
|
||||
type = "virtual_machine"
|
||||
|
||||
@ -73,34 +61,6 @@ class VM(BaseModelConstruct):
|
||||
return {"name": self.name,
|
||||
"type": self.type}
|
||||
|
||||
@property
|
||||
def uptime(self):
|
||||
|
||||
# this NEEDS to be moved to a config file or
|
||||
# possibly be queried from Clerk?
|
||||
tracked_states = [constants.active, constants.building,
|
||||
constants.paused, constants.rescued,
|
||||
constants.resized]
|
||||
|
||||
seconds = self.usage()["state"].uptime(tracked_states)
|
||||
|
||||
# in hours, rounded up:
|
||||
uptime = math.ceil((seconds / 60.0) / 60.0)
|
||||
|
||||
return Decimal(uptime)
|
||||
|
||||
@property
|
||||
def flavor(self):
|
||||
# TODO FIgure out what the hell is going on with ceilometer here,
|
||||
# and why flavor.name isn't always there, and why
|
||||
# sometimes instance_type is needed instead....
|
||||
try:
|
||||
# print "\"flavor.name\" was used"
|
||||
return self._raw["metadata"]["flavor.name"].replace(".", "_")
|
||||
except KeyError:
|
||||
# print "\"instance_type\" was used"
|
||||
return self._raw["metadata"]["instance_type"].replace(".", "_")
|
||||
|
||||
@property
|
||||
def memory(self):
|
||||
return self._raw["metadata"]["memory"]
|
||||
@ -126,33 +86,25 @@ class FloatingIP(BaseModelConstruct):
|
||||
|
||||
relevant_meters = ["ip.floating"]
|
||||
|
||||
usage_strategies = {"duration": {"usage": "duration", "service": "type"}}
|
||||
transformer = transformers.GaugeMax()
|
||||
|
||||
type = "floating_ip" # object storage
|
||||
|
||||
@property
|
||||
def duration(self):
|
||||
return Decimal(self.usage()["ip.floating"].volume())
|
||||
type = "floating_ip"
|
||||
|
||||
|
||||
class Object(BaseModelConstruct):
|
||||
|
||||
relevant_meters = ["storage.objects.size"]
|
||||
|
||||
usage_strategies = {"size": {"usage": "size", "service": "storage_size"}}
|
||||
transformer = transformers.GaugeMax()
|
||||
|
||||
type = "object_storage" # object storage
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
return Decimal(to_mb(self.usage()["storage.objects.size"].volume()))
|
||||
type = "object_storage"
|
||||
|
||||
|
||||
class Volume(BaseModelConstruct):
|
||||
|
||||
relevant_meters = ["volume.size"]
|
||||
|
||||
usage_strategies = {"size": {"usage": "size", "service": "volume_size"}}
|
||||
transformer = transformers.GaugeMax()
|
||||
|
||||
type = "volume"
|
||||
|
||||
@ -161,11 +113,6 @@ class Volume(BaseModelConstruct):
|
||||
return {"name": self.name,
|
||||
"type": self.type}
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
# Size of the thing over time.
|
||||
return Decimal(to_mb(self.usage()["volume.size"].volume()))
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self._raw["metadata"]["display_name"]
|
||||
@ -174,19 +121,6 @@ class Volume(BaseModelConstruct):
|
||||
class Network(BaseModelConstruct):
|
||||
relevant_meters = ["network.outgoing.bytes", "network.incoming.bytes"]
|
||||
|
||||
usage_strategies = {"outgoing": {"usage": "outgoing",
|
||||
"service": "outgoing_megabytes"},
|
||||
"incoming": {"usage": "incoming",
|
||||
"service": "incoming_megabytes"}}
|
||||
transformer = transformers.CumulativeTotal()
|
||||
|
||||
type = "network_interface"
|
||||
|
||||
@property
|
||||
def outgoing(self):
|
||||
# Size of the thing over time.
|
||||
return Decimal(to_mb(self.usage()["network.outgoing.bytes"].volume()))
|
||||
|
||||
@property
|
||||
def incoming(self):
|
||||
# Size of the thing over time.
|
||||
return Decimal(to_mb(self.usage()["network.incoming.bytes"].volume()))
|
||||
|
||||
128
artifice/models/transformers.py
Normal file
128
artifice/models/transformers.py
Normal file
@ -0,0 +1,128 @@
|
||||
import datetime
|
||||
from artifice import constants
|
||||
|
||||
|
||||
class Transformer(object):
|
||||
|
||||
meter_type = None
|
||||
required_meters = []
|
||||
|
||||
def transform_usage(self, meters):
|
||||
self.validate_meters(meters)
|
||||
return self._transform_usage(meters)
|
||||
|
||||
def validate_meters(self, meters):
|
||||
if self.meter_type is None:
|
||||
for meter in meters.values():
|
||||
if meter.name not in self.required_meters:
|
||||
raise AttributeError("Required meters: " +
|
||||
str(self.required_meters))
|
||||
else:
|
||||
for meter in meters.values():
|
||||
if meter.type != self.meter_type:
|
||||
raise AttributeError("Meters must all be of type: " +
|
||||
self.meter_type)
|
||||
|
||||
def _transform_usage(self, meters):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Uptime(Transformer):
|
||||
required_meters = ['state']
|
||||
# required_meters = ['state', 'flavor']
|
||||
|
||||
def _transform_usage(self, meters):
|
||||
# this NEEDS to be moved to a config file
|
||||
tracked_states = [constants.active, constants.building,
|
||||
constants.paused, constants.rescued,
|
||||
constants.resized]
|
||||
usage_dict = {}
|
||||
|
||||
state = meters['state']
|
||||
|
||||
usage = sorted(state.usage(), key=lambda x: x["timestamp"])
|
||||
|
||||
last = usage[0]
|
||||
try:
|
||||
last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
|
||||
constants.date_format)
|
||||
except ValueError:
|
||||
last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
|
||||
constants.other_date_format)
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
uptime = 0.0
|
||||
|
||||
for val in usage[1:]:
|
||||
try:
|
||||
val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
|
||||
constants.date_format)
|
||||
except ValueError:
|
||||
val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
|
||||
constants.other_date_format)
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
if val["counter_volume"] in tracked_states:
|
||||
difference = val["timestamp"] - last["timestamp"]
|
||||
|
||||
uptime = uptime + difference.seconds
|
||||
|
||||
last = val
|
||||
|
||||
usage_dict['flavor1'] = uptime
|
||||
|
||||
return usage_dict
|
||||
|
||||
|
||||
class GaugeMax(Transformer):
|
||||
meter_type = 'gauge'
|
||||
|
||||
def _transform_usage(self, meters):
|
||||
usage_dict = {}
|
||||
for meter in meters.values():
|
||||
usage = meter.usage()
|
||||
max_vol = max([v["counter_volume"] for v in usage])
|
||||
usage_dict[meter.name] = max_vol
|
||||
return usage_dict
|
||||
|
||||
|
||||
class GaugeAverage(Transformer):
|
||||
meter_type = 'gauge'
|
||||
|
||||
def _transform_usage(self, meters):
|
||||
usage_dict = {}
|
||||
for meter in meters.values():
|
||||
usage = meter.usage()
|
||||
length = len(usage)
|
||||
avg_vol = sum([v["counter_volume"] for v in usage]) / length
|
||||
usage_dict[meter.name] = avg_vol
|
||||
return usage_dict
|
||||
|
||||
|
||||
class CumulativeTotal(Transformer):
|
||||
meter_type = 'cumulative'
|
||||
|
||||
def _transform_usage(self, meters):
|
||||
usage_dict = {}
|
||||
for meter in meters.values():
|
||||
measurements = meter.usage()
|
||||
measurements = sorted(measurements, key=lambda x: x["timestamp"])
|
||||
count = 0
|
||||
usage = 0
|
||||
last_measure = None
|
||||
for measure in measurements:
|
||||
if (last_measure is not None and
|
||||
(measure["counter_volume"] <
|
||||
last_measure["counter_volume"])):
|
||||
usage = usage + last_measure["counter_volume"]
|
||||
count = count + 1
|
||||
last_measure = measure
|
||||
|
||||
usage = usage + measurements[-1]["counter_volume"]
|
||||
|
||||
if count > 1:
|
||||
total_usage = usage - measurements[0]["counter_volume"]
|
||||
usage_dict[meter.name] = total_usage
|
||||
return usage_dict
|
||||
@ -19,7 +19,7 @@ class TestDatabaseModule(test_interface.TestInterface):
|
||||
|
||||
count = 0
|
||||
for val in usage.values():
|
||||
for strat in val.usage_strategies:
|
||||
for strat in val.relevant_meters:
|
||||
count += 1
|
||||
|
||||
self.assertEqual(self.session.query(models.UsageEntry).count(), count)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user