Merge "Moving entry sorting to interface" into to_production

distil/interface.py

@@ -15,7 +15,7 @@
 import requests
 import json
 import auth
-from constants import date_format
+from constants import date_format, other_date_format
 import config
 from datetime import timedelta, datetime
 from contextlib import contextmanager
@@ -95,6 +95,21 @@ def add_dates(start, end):
     ]
 
 
+def sort_entries(data):
+    """
+    Setup timestamps as datetime objects,
+    and sort.
+    """
+    for entry in data:
+        try:
+            entry['timestamp'] = datetime.strptime(
+                entry['timestamp'], date_format)
+        except ValueError:
+            entry['timestamp'] = datetime.strptime(
+                entry['timestamp'], other_date_format)
+    return sorted(data, key=lambda x: x['timestamp'])
+
+
 class Tenant(object):
     """A wrapper object for the tenant recieved from keystone."""
     def __init__(self, tenant, conn):
@@ -130,6 +145,6 @@ class Tenant(object):
                 data=json.dumps({'q': fields}))
 
             if r.status_code == 200:
-                return json.loads(r.text)
+                return sort_entries(json.loads(r.text))
             else:
                 raise InterfaceException('%d %s' % (r.status_code, r.text))

distil/transformers.py

@@ -42,8 +42,8 @@ class Uptime(Transformer):
 
         def sort_and_clip_end(usage):
             cleaned = (self._clean_entry(s) for s in usage)
-            clipped = (s for s in cleaned if s['timestamp'] < end)
-            return sorted(clipped, key=lambda x: x['timestamp'])
+            clipped = [s for s in cleaned if s['timestamp'] < end]
+            return clipped
 
         state = sort_and_clip_end(data)
 
@@ -92,14 +92,9 @@ class Uptime(Transformer):
                 'flavor.id', entry['resource_metadata'].get(
                     'instance_flavor_id', 0
                 )
-            )
+            ),
+            'timestamp': entry['timestamp']
         }
-        try:
-            result['timestamp'] = datetime.datetime.strptime(
-                entry['timestamp'], constants.date_format)
-        except ValueError:
-            result['timestamp'] = datetime.datetime.strptime(
-                entry['timestamp'], constants.other_date_format)
         return result
 
 
@@ -183,8 +178,7 @@ class GaugeSum(Transformer):
     def _transform_usage(self, name, data, start, end):
         sum_vol = 0
         for sample in data:
-            t = datetime.datetime.strptime(sample['timestamp'],
-                                           '%Y-%m-%dT%H:%M:%S.%f')
+            t = sample['timestamp']
             if t >= start and t < end:
                 sum_vol += sample["counter_volume"]
         return {name: sum_vol}