pulled in datetime/decimal code, fixed up json encoding, started tying into disk_storage in rolling test

shoebox/disk_storage.py

@@ -3,15 +3,6 @@ import datetime
 import json
 import struct
 
-class DatetimeEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, datetime.datetime):
-            if obj.utcoffset() is not None:
-                obj = obj - obj.utcoffset()
-            return int(calendar.timegm(obj.timetuple()) * 1000 +
-                         obj.microsecond / 1000)
-        return super(DatetimeEncoder, self).default(obj)
-
 
 class InvalidVersion(Exception):
     pass

shoebox/roll_checker.py

@@ -48,5 +48,5 @@ class SizeRollChecker(RollChecker):
         self.size_in_gb = size_in_gb
 
     def check(self, archive):
-        size = archive._get_file_handle().tell()
+        size = archive.get_file_handle().tell()
         return size / 1073741824 >= self.size_in_gb

shoebox/utils.py

@@ -13,9 +13,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import calendar
 import datetime
+import decimal
+import json
 
 
 def now():
     """Broken out for testing."""
     return datetime.datetime.utcnow()
+
+
+def dt_to_decimal(utc):
+    decimal.getcontext().prec = 30
+    return decimal.Decimal(str(calendar.timegm(utc.utctimetuple()))) + \
+           (decimal.Decimal(str(utc.microsecond)) /
+           decimal.Decimal("1000000.0"))
+
+
+def dt_from_decimal(dec):
+    if dec == None:
+        return "n/a"
+    integer = int(dec)
+    micro = (dec - decimal.Decimal(integer)) * decimal.Decimal(1000000)
+
+    daittyme = datetime.datetime.utcfromtimestamp(integer)
+    return daittyme.replace(microsecond=micro)
+
+
+class DateTimeEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, datetime.datetime):
+            if obj.utcoffset() is not None:
+                obj = obj - obj.utcoffset()
+            return str(dt_to_decimal(obj))
+        return super(DateTimeEncoder, self).default(obj)

test/integration/gen_events.py

@@ -296,7 +296,7 @@ if __name__ == '__main__':
     now = datetime.datetime.utcnow()
     start = now
     nevents = 0
-    while nevents < 100:
+    while nevents < 10000:
         e = g.generate(now)
         if e:
             nevents += len(e)

test/integration/test_rolling.py

@@ -1,10 +1,12 @@
 import datetime
+import json
 import mock
 import os
 import shutil
 import unittest
 
 
+from shoebox import disk_storage
 from shoebox import roll_checker
 from shoebox import roll_manager
 from shoebox import utils
@@ -32,7 +34,18 @@ class TestSizeRolling(unittest.TestCase):
         nevents = 0
         now = datetime.datetime.utcnow()
         while nevents < 1000:
-            e = g.generate(now)
+            events = g.generate(now)
-            if e:
+            if events:
-                nevents += len(e)
+                nevents += len(events)
+                for event in events:
+                    metadata = {'event': event['event'],
+                                'request_id': event['request_id'],
+                                'generated': str(event['when']),
+                                'uuid': event['uuid'],
+                                }
+                    json_event = json.dumps(event,
+                                            cls=utils.DateTimeEncoder)
+                    manager.write(metadata, json_event)
+
             now = g.move_to_next_tick(now)
+

test/test_disk_storage.py

@@ -9,25 +9,6 @@ import dateutil.tz
 from shoebox import disk_storage
 
 
-class TestDiskStorage(unittest.TestCase):
-    def setUp(self):
-        self.handler = disk_storage.DatetimeEncoder()
-
-    def test_handle_datetime_non_datetime(self):
-        self.assertRaises(TypeError, self.handler.default, "text")
-
-    def test_handle_datetime(self):
-        now = datetime.datetime(day=1, month=2, year=2014,
-                                hour=10, minute=11, second=12)
-        self.assertEqual(1391249472000, self.handler.default(now))
-
-    def test_handle_datetime_offset(self):
-        now = datetime.datetime(day=1, month=2, year=2014,
-                                hour=10, minute=11, second=12,
-                                tzinfo=dateutil.tz.tzoffset(None, 4*60*60))
-        self.assertEqual(1391235072000, self.handler.default(now))
-
-
 class TestVersion0(unittest.TestCase):
     def setUp(self):
        self.v0 = disk_storage.Version0()

test/test_roll_checker.py

@@ -40,11 +40,11 @@ class TestRollChecker(unittest.TestCase):
         x = roll_checker.SizeRollChecker(10)
 
         archive = mock.Mock()
-        archive._get_file_handle.return_value.tell.return_value = one_gig * 5
+        archive.get_file_handle.return_value.tell.return_value = one_gig * 5
         self.assertFalse(x.check(archive))
 
-        archive._get_file_handle.return_value.tell.return_value = one_gig * 10
+        archive.get_file_handle.return_value.tell.return_value = one_gig * 10
         self.assertTrue(x.check(archive))
 
-        archive._get_file_handle.return_value.tell.return_value = one_gig * 11
+        archive.get_file_handle.return_value.tell.return_value = one_gig * 11
         self.assertTrue(x.check(archive))

tox.ini

@@ -8,4 +8,4 @@ deps =
     mock
     coverage
 
-commands = nosetests -d --with-coverage --cover-inclusive --cover-package shoebox []
+commands = nosetests -d -v --with-coverage --cover-inclusive --cover-package shoebox []