diff --git a/shoebox/disk_storage.py b/shoebox/disk_storage.py index e9645c1..6497f68 100644 --- a/shoebox/disk_storage.py +++ b/shoebox/disk_storage.py @@ -3,15 +3,6 @@ import datetime import json import struct -class DatetimeEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, datetime.datetime): - if obj.utcoffset() is not None: - obj = obj - obj.utcoffset() - return int(calendar.timegm(obj.timetuple()) * 1000 + - obj.microsecond / 1000) - return super(DatetimeEncoder, self).default(obj) - class InvalidVersion(Exception): pass diff --git a/shoebox/roll_checker.py b/shoebox/roll_checker.py index edad3e4..784d777 100644 --- a/shoebox/roll_checker.py +++ b/shoebox/roll_checker.py @@ -48,5 +48,5 @@ class SizeRollChecker(RollChecker): self.size_in_gb = size_in_gb def check(self, archive): - size = archive._get_file_handle().tell() + size = archive.get_file_handle().tell() return size / 1073741824 >= self.size_in_gb diff --git a/shoebox/utils.py b/shoebox/utils.py index b6b72ae..59220ce 100644 --- a/shoebox/utils.py +++ b/shoebox/utils.py @@ -13,9 +13,38 @@ # See the License for the specific language governing permissions and # limitations under the License. +import calendar import datetime +import decimal +import json def now(): """Broken out for testing.""" return datetime.datetime.utcnow() + + +def dt_to_decimal(utc): + decimal.getcontext().prec = 30 + return decimal.Decimal(str(calendar.timegm(utc.utctimetuple()))) + \ + (decimal.Decimal(str(utc.microsecond)) / + decimal.Decimal("1000000.0")) + + +def dt_from_decimal(dec): + if dec == None: + return "n/a" + integer = int(dec) + micro = (dec - decimal.Decimal(integer)) * decimal.Decimal(1000000) + + daittyme = datetime.datetime.utcfromtimestamp(integer) + return daittyme.replace(microsecond=micro) + + +class DateTimeEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, datetime.datetime): + if obj.utcoffset() is not None: + obj = obj - obj.utcoffset() + return str(dt_to_decimal(obj)) + return super(DateTimeEncoder, self).default(obj) diff --git a/test/integration/gen_events.py b/test/integration/gen_events.py index 20b6c04..90fecb9 100644 --- a/test/integration/gen_events.py +++ b/test/integration/gen_events.py @@ -296,7 +296,7 @@ if __name__ == '__main__': now = datetime.datetime.utcnow() start = now nevents = 0 - while nevents < 100: + while nevents < 10000: e = g.generate(now) if e: nevents += len(e) diff --git a/test/integration/test_rolling.py b/test/integration/test_rolling.py index d7be2c9..7f89601 100644 --- a/test/integration/test_rolling.py +++ b/test/integration/test_rolling.py @@ -1,10 +1,12 @@ import datetime +import json import mock import os import shutil import unittest +from shoebox import disk_storage from shoebox import roll_checker from shoebox import roll_manager from shoebox import utils @@ -32,7 +34,18 @@ class TestSizeRolling(unittest.TestCase): nevents = 0 now = datetime.datetime.utcnow() while nevents < 1000: - e = g.generate(now) - if e: - nevents += len(e) + events = g.generate(now) + if events: + nevents += len(events) + for event in events: + metadata = {'event': event['event'], + 'request_id': event['request_id'], + 'generated': str(event['when']), + 'uuid': event['uuid'], + } + json_event = json.dumps(event, + cls=utils.DateTimeEncoder) + manager.write(metadata, json_event) + now = g.move_to_next_tick(now) + diff --git a/test/test_disk_storage.py b/test/test_disk_storage.py index 9a8efe2..cdcb7a0 100644 --- a/test/test_disk_storage.py +++ b/test/test_disk_storage.py @@ -9,25 +9,6 @@ import dateutil.tz from shoebox import disk_storage -class TestDiskStorage(unittest.TestCase): - def setUp(self): - self.handler = disk_storage.DatetimeEncoder() - - def test_handle_datetime_non_datetime(self): - self.assertRaises(TypeError, self.handler.default, "text") - - def test_handle_datetime(self): - now = datetime.datetime(day=1, month=2, year=2014, - hour=10, minute=11, second=12) - self.assertEqual(1391249472000, self.handler.default(now)) - - def test_handle_datetime_offset(self): - now = datetime.datetime(day=1, month=2, year=2014, - hour=10, minute=11, second=12, - tzinfo=dateutil.tz.tzoffset(None, 4*60*60)) - self.assertEqual(1391235072000, self.handler.default(now)) - - class TestVersion0(unittest.TestCase): def setUp(self): self.v0 = disk_storage.Version0() diff --git a/test/test_roll_checker.py b/test/test_roll_checker.py index 9e911a0..0289092 100644 --- a/test/test_roll_checker.py +++ b/test/test_roll_checker.py @@ -40,11 +40,11 @@ class TestRollChecker(unittest.TestCase): x = roll_checker.SizeRollChecker(10) archive = mock.Mock() - archive._get_file_handle.return_value.tell.return_value = one_gig * 5 + archive.get_file_handle.return_value.tell.return_value = one_gig * 5 self.assertFalse(x.check(archive)) - archive._get_file_handle.return_value.tell.return_value = one_gig * 10 + archive.get_file_handle.return_value.tell.return_value = one_gig * 10 self.assertTrue(x.check(archive)) - archive._get_file_handle.return_value.tell.return_value = one_gig * 11 + archive.get_file_handle.return_value.tell.return_value = one_gig * 11 self.assertTrue(x.check(archive)) diff --git a/tox.ini b/tox.ini index 9118d5a..f69bb33 100644 --- a/tox.ini +++ b/tox.ini @@ -8,4 +8,4 @@ deps = mock coverage -commands = nosetests -d --with-coverage --cover-inclusive --cover-package shoebox [] +commands = nosetests -d -v --with-coverage --cover-inclusive --cover-package shoebox []