From e426aa5e84d939249bbbf6d9cb5dbe09783710e7 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 22 Feb 2013 13:41:00 -0500 Subject: [PATCH] Adding dbapi query limit and offset --- stacktach/dbapi.py | 59 +++++++++++-------- stacktach/models.py | 20 +++---- tests/unit/test_dbapi.py | 117 ++++++++++++++++++++++++++++++++++++++ tests/unit/test_worker.py | 1 + 4 files changed, 163 insertions(+), 34 deletions(-) diff --git a/stacktach/dbapi.py b/stacktach/dbapi.py index 98003a8..1b2cbb1 100644 --- a/stacktach/dbapi.py +++ b/stacktach/dbapi.py @@ -52,14 +52,8 @@ def api_call(func): @api_call def list_usage_launches(request): - filter_args = _get_filter_args(models.InstanceUsage, request) - - if len(filter_args) > 0: - objects = models.InstanceUsage.objects.filter(**filter_args) - else: - objects = models.InstanceUsage.objects.all() - - dicts = _convert_model_list(objects.order_by("launched_at")) + objects = get_db_objects(models.InstanceUsage, request, 'launched_at') + dicts = _convert_model_list(objects) return {'launches': dicts} @@ -70,14 +64,8 @@ def get_usage_launch(request, launch_id): @api_call def list_usage_deletes(request): - filter_args = _get_filter_args(models.InstanceDeletes, request) - - if len(filter_args) > 0: - objects = models.InstanceDeletes.objects.filter(**filter_args) - else: - objects = models.InstanceDeletes.objects.all() - - dicts = _convert_model_list(objects.order_by("launched_at")) + objects = get_db_objects(models.InstanceDeletes, request, 'launched_at') + dicts = _convert_model_list(objects) return {'deletes': dicts} @@ -88,14 +76,8 @@ def get_usage_delete(request, delete_id): @api_call def list_usage_exists(request): - filter_args = _get_filter_args(models.InstanceExists, request) - - if len(filter_args) > 0: - objects = models.InstanceExists.objects.filter(**filter_args) - else: - objects = models.InstanceExists.objects.all() - - dicts = _convert_model_list(objects.order_by("id")) + objects = get_db_objects(models.InstanceExists, request, 'id') + dicts = _convert_model_list(objects) return {'exists': dicts} @@ -145,6 +127,35 @@ def _get_filter_args(klass, request): return filter_args +def get_db_objects(klass, request, default_order_by, direction='asc'): + filter_args = _get_filter_args(klass, request) + + if len(filter_args) > 0: + objects = klass.objects.filter(**filter_args) + else: + objects = klass.objects.all() + + order_by = request.GET.get('order_by', default_order_by) + _check_has_field(klass, order_by) + + direction = request.GET.get('direction', direction) + if direction == 'desc': + order_by = '-%s' % order_by + + offset = request.GET.get('offset') + limit = request.GET.get('limit') + if offset: + start = int(offset) + else: + start = None + offset = 0 + if limit: + end = int(offset) + int(limit) + else: + end = None + return objects.order_by(order_by)[start:end] + + def _convert_model(model): model_dict = model_to_dict(model) for key in model_dict: diff --git a/stacktach/models.py b/stacktach/models.py index 9a09059..222a933 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -87,11 +87,11 @@ class InstanceUsage(models.Model): class InstanceDeletes(models.Model): instance = models.CharField(max_length=50, null=True, - blank=True, db_index=True) + blank=True, db_index=True) launched_at = models.DecimalField(null=True, max_digits=20, - decimal_places=6) + decimal_places=6) deleted_at = models.DecimalField(null=True, max_digits=20, - decimal_places=6) + decimal_places=6) raw = models.ForeignKey(RawData, null=True) @@ -109,13 +109,13 @@ class InstanceExists(models.Model): launched_at = models.DecimalField(null=True, max_digits=20, decimal_places=6) deleted_at = models.DecimalField(null=True, max_digits=20, - decimal_places=6) - message_id = models.CharField(max_length=50, null=True, - blank=True, db_index=True) - instance_type_id = models.CharField(max_length=50, - null=True, - blank=True, - db_index=True) + decimal_places=6) + message_id = models.CharField(max_length=50, null=True, + blank=True, db_index=True) + instance_type_id = models.CharField(max_length=50, + null=True, + blank=True, + db_index=True) status = models.CharField(max_length=50, db_index=True, choices=STATUS_CHOICES, default=PENDING) diff --git a/tests/unit/test_dbapi.py b/tests/unit/test_dbapi.py index 8d18e08..3f993d3 100644 --- a/tests/unit/test_dbapi.py +++ b/tests/unit/test_dbapi.py @@ -20,6 +20,8 @@ class StacktachRawParsingTestCase(unittest.TestCase): fake_model = self.mox.CreateMockAnything() fake_meta = self.mox.CreateMockAnything() fake_model._meta = fake_meta + fake_orm = self.mox.CreateMockAnything() + fake_model.objects = fake_orm return fake_model def test_get_filter_args(self): @@ -82,3 +84,118 @@ class StacktachRawParsingTestCase(unittest.TestCase): fake_model, fake_request) self.mox.VerifyAll() + + def test_get_db_objects(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {} + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn({}) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.all().AndReturn(result) + result.order_by('id').AndReturn(result) + result.__getitem__(slice(None, None, None)).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() + + def test_get_db_objects_desc(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'direction': 'desc'} + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn({}) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.all().AndReturn(result) + result.order_by('-id').AndReturn(result) + result.__getitem__(slice(None, None, None)).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() + + def test_get_db_objects_limit(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'limit': 1} + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn({}) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.all().AndReturn(result) + result.order_by('id').AndReturn(result) + result.__getitem__(slice(None, 1, None)).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() + + def test_get_db_objects_offset(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'offset': 1} + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn({}) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.all().AndReturn(result) + result.order_by('id').AndReturn(result) + result.__getitem__(slice(1, None, None)).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() + + def test_get_db_objects_offset_and_limit(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'offset': 2, 'limit': 2} + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn({}) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.all().AndReturn(result) + result.order_by('id').AndReturn(result) + result.__getslice__(2, 4).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() + + def test_get_db_objects_with_filter(self): + fake_model = self.make_fake_model() + fake_request = self.mox.CreateMockAnything() + filters = {'instance': INSTANCE_ID_1} + fake_request.GET = filters + self.mox.StubOutWithMock(dbapi, '_get_filter_args') + dbapi._get_filter_args(fake_model, fake_request).AndReturn(filters) + self.mox.StubOutWithMock(dbapi, '_check_has_field') + dbapi._check_has_field(fake_model, 'id') + result = self.mox.CreateMockAnything() + fake_model.objects.filter(**filters).AndReturn(result) + result.order_by('id').AndReturn(result) + result.__getitem__(slice(None, None, None)).AndReturn(result) + self.mox.ReplayAll() + + query_result = dbapi.get_db_objects(fake_model, fake_request, 'id') + self.assertEquals(query_result, result) + + self.mox.VerifyAll() diff --git a/tests/unit/test_worker.py b/tests/unit/test_worker.py index ada86bb..3d2d3f5 100644 --- a/tests/unit/test_worker.py +++ b/tests/unit/test_worker.py @@ -9,6 +9,7 @@ import mox from stacktach import db, views import worker.worker as worker + class NovaConsumerTestCase(unittest.TestCase): def setUp(self): self.mox = mox.Mox()