First implementation of transformers. Seems to work, all old tests pass.

Uptime doesn't use flavor metric yet.

Transformers need testing.

artifice/artifact.py

@@ -1,154 +0,0 @@
-import datetime
-from constants import date_format, other_date_format
-
-
-class Artifact(object):
-
-    """
-    Provides base artifact controls; generic typing information
-    for the artifact structures.
-    """
-
-    def __init__(self, resource, usage, start, end):
-
-        self.resource = resource
-        self.usage = usage  # Raw meter data from Ceilometer
-        self.start = start
-        self.end = end
-
-    def __getitem__(self, item):
-        if item in self._data:
-            return self._data[item]
-        raise KeyError("no such item %s" % item)
-
-    def volume(self):
-        """
-        Default billable number for this volume
-        """
-        return sum([x["counter_volume"] for x in self.usage])
-
-
-class Cumulative(Artifact):
-
-    def volume(self):
-        measurements = self.usage
-        measurements = sorted(measurements, key=lambda x: x["timestamp"])
-        count = 0
-        usage = 0
-        last_measure = None
-        for measure in measurements:
-            if last_measure is not None and (measure["counter_volume"] <
-                                             last_measure["counter_volume"]):
-                usage = usage + last_measure["counter_volume"]
-            count = count + 1
-            last_measure = measure
-
-        usage = usage + measurements[-1]["counter_volume"]
-
-        if count > 1:
-            total_usage = usage - measurements[0]["counter_volume"]
-        return total_usage
-
-
-# Gauge and Delta have very little to do: They are expected only to
-# exist as "not a cumulative" sort of artifact.
-class Gauge(Artifact):
-
-    def volume(self):
-        """
-        Default billable number for this volume
-        """
-        # print "Usage is %s" % self.usage
-        usage = sorted(self.usage, key=lambda x: x["timestamp"])
-
-        blocks = []
-        curr = [usage[0]]
-        last = usage[0]
-        try:
-            last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
-                                                           date_format)
-        except ValueError:
-            last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
-                                                           other_date_format)
-        except TypeError:
-            pass
-
-        for val in usage[1:]:
-            try:
-                val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
-                                                              date_format)
-            except ValueError:
-                val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
-                                                              other_date_format)
-            except TypeError:
-                pass
-
-            difference = (val['timestamp'] - last["timestamp"])
-            if difference > datetime.timedelta(hours=1):
-                blocks.append(curr)
-                curr = [val]
-                last = val
-            else:
-                curr.append(val)
-
-        # this adds the last remaining values as a block of their own on exit
-        # might mean people are billed twice for an hour at times...
-        # but solves the issue of not billing if there isn't enough data for
-        # full hour.
-        blocks.append(curr)
-
-        # We are now sorted into 1-hour blocks
-        totals = []
-        for block in blocks:
-            usage = max([v["counter_volume"] for v in block])
-            totals.append(usage)
-
-        # totals = [max(x, key=lambda val: val["counter_volume"] ) for x in blocks]
-        # totals is now an array of max values per hour for a given month.
-        return sum(totals)
-
-    # This continues to be wrong.
-    def uptime(self, tracked):
-        """Calculates uptime accurately for the given 'tracked' states.
-        - Will ignore all other states.
-        - Relies heavily on the existence of a state meter, and
-          should only ever be called on the state meter.
-
-        Returns: uptime in seconds"""
-
-        usage = sorted(self.usage, key=lambda x: x["timestamp"])
-
-        last = usage[0]
-        try:
-            last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
-                                                           date_format)
-        except ValueError:
-            last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
-                                                           other_date_format)
-        except TypeError:
-            pass
-
-        uptime = 0.0
-
-        for val in usage[1:]:
-            try:
-                val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
-                                                              date_format)
-            except ValueError:
-                val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
-                                                              other_date_format)
-            except TypeError:
-                pass
-
-            if val["counter_volume"] in tracked:
-                difference = val["timestamp"] - last["timestamp"]
-
-                uptime = uptime + difference.seconds
-
-            last = val
-
-        return uptime
-
-
-class Delta(Artifact):
-    pass

artifice/database.py

@@ -30,13 +30,7 @@ class Database(object):
            in a resource, for all the resources given"""
 
         for resource in usage:
-            for key in resource.usage_strategies:
+            for service, volume in resource.usage().iteritems():
-                strategy = resource.usage_strategies[key]
-                volume = resource.get(strategy['usage'])
-                try:
-                    service = resource.get(strategy['service'])
-                except AttributeError:
-                    service = strategy['service']
                 resource_id = resource.get("resource_id")
                 tenant_id = resource.get("tenant_id")
 

artifice/interface.py

@@ -1,7 +1,5 @@
 import requests
 import json
-from collections import defaultdict
-import artifact
 import auth
 from ceilometerclient.v2.client import Client as ceilometer
 from .models import resources
@@ -275,7 +273,27 @@ class Meter(object):
         self.conn = conn
         self.start = start
         self.end = end
-        # self.meter = meter
+
+        self.measurements = self.get_meter(start, end,
+                                           self.conn.auth.auth_token)
+
+        self.type = set([a["counter_type"] for a in self.measurements])
+        if len(self.type) > 1:
+            # That's a big problem
+            raise RuntimeError("Too many types for measurement!")
+        elif len(self.type) == 0:
+            raise RuntimeError("No types!")
+        else:
+            self.type = self.type.pop()
+
+        self.name = set([a["counter_name"] for a in self.measurements])
+        if len(self.name) > 1:
+            # That's a big problem
+            raise RuntimeError("Too many names for measurement!")
+        elif len(self.name) == 0:
+            raise RuntimeError("No types!")
+        else:
+            self.name = self.name.pop()
 
     def get_meter(self, start, end, auth):
         # Meter is a href; in this case, it has a set of fields with it already.
@@ -294,35 +312,5 @@ class Meter(object):
         )
         return json.loads(r.text)
 
-    def volume(self):
+    def usage(self):
-        return self.usage(self.start, self.end)
+        return self.measurements
-
-    def usage(self, start, end):
-        """
-        Usage condenses the entirety of a meter into a single datapoint:
-        A volume value that we can plot as a single number against previous
-        usage for a given range.
-        """
-        measurements = self.get_meter(start, end, self.conn.auth.auth_token)
-        # return measurements
-
-        # print measurements
-
-        # self.measurements = defaultdict(list)
-        self.type = set([a["counter_type"] for a in measurements])
-        if len(self.type) > 1:
-            # That's a big problem
-            raise RuntimeError("Too many types for measurement!")
-        elif len(self.type) == 0:
-            raise RuntimeError("No types!")
-        else:
-            self.type = self.type.pop()
-        type_ = None
-        if self.type == "cumulative":
-            type_ = artifact.Cumulative
-        elif self.type == "gauge":
-            type_ = artifact.Gauge
-        elif self.type == "delta":
-            type_ = artifact.Delta
-
-        return type_(self.resource, measurements, start, end)

artifice/models/resources.py

@@ -1,7 +1,4 @@
-from decimal import Decimal
+from . import transformers
-import math
-
-from artifice import constants
 
 
 class BaseModelConstruct(object):
@@ -34,37 +31,28 @@ class BaseModelConstruct(object):
         # information based on a meter, I guess?
         return getattr(self, name)
 
-    def usage(self):
+    def meters(self):
         dct = {}
         for meter in self.relevant_meters:
             try:
-                vol = self._raw.meter(meter, self.start, self.end).volume()
+                mtr = self._raw.meter(meter, self.start, self.end)
-                dct[meter] = vol
+                dct[meter] = mtr
             except AttributeError:
                 # This is OK. We're not worried about non-existent meters,
                 # I think. For now, anyway.
                 pass
         return dct
 
-    def save(self):
+    def usage(self):
-        for meter in self.relevant_meters:
+        meters = self.meters()
-            try:
+        usage = self.transformer.transform_usage(meters)
-                self._raw.meter(meter, self.start, self.end).save()
+        return usage
-            except AttributeError:
-                # This is OK. We're not worried about non-existent meters,
-                # I think. For now, anyway.
-                pass
-
-
-def to_mb(bytes):
-    # function to make code easier to understand elsewhere.
-    return (bytes / 1000) / 1000
 
 
 class VM(BaseModelConstruct):
     relevant_meters = ["state"]
 
-    usage_strategies = {"uptime": {"usage": "uptime", "service": "flavor"}}
+    transformer = transformers.Uptime()
 
     type = "virtual_machine"
 
@@ -73,34 +61,6 @@ class VM(BaseModelConstruct):
         return {"name": self.name,
                 "type": self.type}
 
-    @property
-    def uptime(self):
-
-        # this NEEDS to be moved to a config file or
-        # possibly be queried from Clerk?
-        tracked_states = [constants.active, constants.building,
-                          constants.paused, constants.rescued,
-                          constants.resized]
-
-        seconds = self.usage()["state"].uptime(tracked_states)
-
-        # in hours, rounded up:
-        uptime = math.ceil((seconds / 60.0) / 60.0)
-
-        return Decimal(uptime)
-
-    @property
-    def flavor(self):
-        # TODO FIgure out what the hell is going on with ceilometer here,
-        # and why flavor.name isn't always there, and why
-        # sometimes instance_type is needed instead....
-        try:
-            # print "\"flavor.name\" was used"
-            return self._raw["metadata"]["flavor.name"].replace(".", "_")
-        except KeyError:
-            # print "\"instance_type\" was used"
-            return self._raw["metadata"]["instance_type"].replace(".", "_")
-
     @property
     def memory(self):
         return self._raw["metadata"]["memory"]
@@ -126,33 +86,25 @@ class FloatingIP(BaseModelConstruct):
 
     relevant_meters = ["ip.floating"]
 
-    usage_strategies = {"duration": {"usage": "duration", "service": "type"}}
+    transformer = transformers.GaugeMax()
 
-    type = "floating_ip"  # object storage
+    type = "floating_ip"
-
-    @property
-    def duration(self):
-        return Decimal(self.usage()["ip.floating"].volume())
 
 
 class Object(BaseModelConstruct):
 
     relevant_meters = ["storage.objects.size"]
 
-    usage_strategies = {"size": {"usage": "size", "service": "storage_size"}}
+    transformer = transformers.GaugeMax()
 
-    type = "object_storage"  # object storage
+    type = "object_storage"
-
-    @property
-    def size(self):
-        return Decimal(to_mb(self.usage()["storage.objects.size"].volume()))
 
 
 class Volume(BaseModelConstruct):
 
     relevant_meters = ["volume.size"]
 
-    usage_strategies = {"size": {"usage": "size", "service": "volume_size"}}
+    transformer = transformers.GaugeMax()
 
     type = "volume"
 
@@ -161,11 +113,6 @@ class Volume(BaseModelConstruct):
         return {"name": self.name,
                 "type": self.type}
 
-    @property
-    def size(self):
-        # Size of the thing over time.
-        return Decimal(to_mb(self.usage()["volume.size"].volume()))
-
     @property
     def name(self):
         return self._raw["metadata"]["display_name"]
@@ -174,19 +121,6 @@ class Volume(BaseModelConstruct):
 class Network(BaseModelConstruct):
     relevant_meters = ["network.outgoing.bytes", "network.incoming.bytes"]
 
-    usage_strategies = {"outgoing": {"usage": "outgoing",
+    transformer = transformers.CumulativeTotal()
-                                     "service": "outgoing_megabytes"},
-                        "incoming": {"usage": "incoming",
-                                     "service": "incoming_megabytes"}}
 
     type = "network_interface"
-
-    @property
-    def outgoing(self):
-        # Size of the thing over time.
-        return Decimal(to_mb(self.usage()["network.outgoing.bytes"].volume()))
-
-    @property
-    def incoming(self):
-        # Size of the thing over time.
-        return Decimal(to_mb(self.usage()["network.incoming.bytes"].volume()))

artifice/models/transformers.py

@@ -0,0 +1,128 @@
+import datetime
+from artifice import constants
+
+
+class Transformer(object):
+
+    meter_type = None
+    required_meters = []
+
+    def transform_usage(self, meters):
+        self.validate_meters(meters)
+        return self._transform_usage(meters)
+
+    def validate_meters(self, meters):
+        if self.meter_type is None:
+            for meter in meters.values():
+                if meter.name not in self.required_meters:
+                    raise AttributeError("Required meters: " +
+                                         str(self.required_meters))
+        else:
+            for meter in meters.values():
+                if meter.type != self.meter_type:
+                    raise AttributeError("Meters must all be of type: " +
+                                         self.meter_type)
+
+    def _transform_usage(self, meters):
+        raise NotImplementedError
+
+
+class Uptime(Transformer):
+    required_meters = ['state']
+    # required_meters = ['state', 'flavor']
+
+    def _transform_usage(self, meters):
+        # this NEEDS to be moved to a config file
+        tracked_states = [constants.active, constants.building,
+                          constants.paused, constants.rescued,
+                          constants.resized]
+        usage_dict = {}
+
+        state = meters['state']
+
+        usage = sorted(state.usage(), key=lambda x: x["timestamp"])
+
+        last = usage[0]
+        try:
+            last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
+                                                           constants.date_format)
+        except ValueError:
+            last["timestamp"] = datetime.datetime.strptime(last["timestamp"],
+                                                           constants.other_date_format)
+        except TypeError:
+            pass
+
+        uptime = 0.0
+
+        for val in usage[1:]:
+            try:
+                val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
+                                                              constants.date_format)
+            except ValueError:
+                val["timestamp"] = datetime.datetime.strptime(val["timestamp"],
+                                                              constants.other_date_format)
+            except TypeError:
+                pass
+
+            if val["counter_volume"] in tracked_states:
+                difference = val["timestamp"] - last["timestamp"]
+
+                uptime = uptime + difference.seconds
+
+            last = val
+
+        usage_dict['flavor1'] = uptime
+
+        return usage_dict
+
+
+class GaugeMax(Transformer):
+    meter_type = 'gauge'
+
+    def _transform_usage(self, meters):
+        usage_dict = {}
+        for meter in meters.values():
+            usage = meter.usage()
+            max_vol = max([v["counter_volume"] for v in usage])
+            usage_dict[meter.name] = max_vol
+        return usage_dict
+
+
+class GaugeAverage(Transformer):
+    meter_type = 'gauge'
+
+    def _transform_usage(self, meters):
+        usage_dict = {}
+        for meter in meters.values():
+            usage = meter.usage()
+            length = len(usage)
+            avg_vol = sum([v["counter_volume"] for v in usage]) / length
+            usage_dict[meter.name] = avg_vol
+        return usage_dict
+
+
+class CumulativeTotal(Transformer):
+    meter_type = 'cumulative'
+
+    def _transform_usage(self, meters):
+        usage_dict = {}
+        for meter in meters.values():
+            measurements = meter.usage()
+            measurements = sorted(measurements, key=lambda x: x["timestamp"])
+            count = 0
+            usage = 0
+            last_measure = None
+            for measure in measurements:
+                if (last_measure is not None and
+                        (measure["counter_volume"] <
+                            last_measure["counter_volume"])):
+                    usage = usage + last_measure["counter_volume"]
+                count = count + 1
+                last_measure = measure
+
+            usage = usage + measurements[-1]["counter_volume"]
+
+            if count > 1:
+                total_usage = usage - measurements[0]["counter_volume"]
+                usage_dict[meter.name] = total_usage
+        return usage_dict

tests/test_database_module.py

@@ -19,7 +19,7 @@ class TestDatabaseModule(test_interface.TestInterface):
 
         count = 0
         for val in usage.values():
-            for strat in val.usage_strategies:
+            for strat in val.relevant_meters:
                 count += 1
 
         self.assertEqual(self.session.query(models.UsageEntry).count(), count)