Adding dbapi query limit and offset

stacktach/dbapi.py

@@ -52,14 +52,8 @@ def api_call(func):
 
 @api_call
 def list_usage_launches(request):
-    filter_args = _get_filter_args(models.InstanceUsage, request)
+    objects = get_db_objects(models.InstanceUsage, request, 'launched_at')
-
+    dicts = _convert_model_list(objects)
-    if len(filter_args) > 0:
-        objects = models.InstanceUsage.objects.filter(**filter_args)
-    else:
-        objects = models.InstanceUsage.objects.all()
-
-    dicts = _convert_model_list(objects.order_by("launched_at"))
     return {'launches': dicts}
 
 
@@ -70,14 +64,8 @@ def get_usage_launch(request, launch_id):
 
 @api_call
 def list_usage_deletes(request):
-    filter_args = _get_filter_args(models.InstanceDeletes, request)
+    objects = get_db_objects(models.InstanceDeletes, request, 'launched_at')
-
+    dicts = _convert_model_list(objects)
-    if len(filter_args) > 0:
-        objects = models.InstanceDeletes.objects.filter(**filter_args)
-    else:
-        objects = models.InstanceDeletes.objects.all()
-
-    dicts = _convert_model_list(objects.order_by("launched_at"))
     return {'deletes': dicts}
 
 
@@ -88,14 +76,8 @@ def get_usage_delete(request, delete_id):
 
 @api_call
 def list_usage_exists(request):
-    filter_args = _get_filter_args(models.InstanceExists, request)
+    objects = get_db_objects(models.InstanceExists, request, 'id')
-
+    dicts = _convert_model_list(objects)
-    if len(filter_args) > 0:
-        objects = models.InstanceExists.objects.filter(**filter_args)
-    else:
-        objects = models.InstanceExists.objects.all()
-
-    dicts = _convert_model_list(objects.order_by("id"))
     return {'exists': dicts}
 
 
@@ -145,6 +127,35 @@ def _get_filter_args(klass, request):
     return filter_args
 
 
+def get_db_objects(klass, request, default_order_by, direction='asc'):
+    filter_args = _get_filter_args(klass, request)
+
+    if len(filter_args) > 0:
+        objects = klass.objects.filter(**filter_args)
+    else:
+        objects = klass.objects.all()
+
+    order_by = request.GET.get('order_by', default_order_by)
+    _check_has_field(klass, order_by)
+
+    direction = request.GET.get('direction', direction)
+    if direction == 'desc':
+        order_by = '-%s' % order_by
+
+    offset = request.GET.get('offset')
+    limit = request.GET.get('limit')
+    if offset:
+        start = int(offset)
+    else:
+        start = None
+        offset = 0
+    if limit:
+        end = int(offset) + int(limit)
+    else:
+        end = None
+    return objects.order_by(order_by)[start:end]
+
+
 def _convert_model(model):
     model_dict = model_to_dict(model)
     for key in model_dict:

stacktach/models.py

@@ -87,11 +87,11 @@ class InstanceUsage(models.Model):
 
 class InstanceDeletes(models.Model):
     instance = models.CharField(max_length=50, null=True,
-        blank=True, db_index=True)
+                                blank=True, db_index=True)
     launched_at = models.DecimalField(null=True, max_digits=20,
-        decimal_places=6)
+                                      decimal_places=6)
     deleted_at = models.DecimalField(null=True, max_digits=20,
-        decimal_places=6)
+                                     decimal_places=6)
     raw = models.ForeignKey(RawData, null=True)
 
 
@@ -109,13 +109,13 @@ class InstanceExists(models.Model):
     launched_at = models.DecimalField(null=True, max_digits=20,
                                       decimal_places=6)
     deleted_at = models.DecimalField(null=True, max_digits=20,
-                                      decimal_places=6)
+                                     decimal_places=6)
-    message_id =  models.CharField(max_length=50, null=True,
+    message_id = models.CharField(max_length=50, null=True,
-                                   blank=True, db_index=True)
+                                  blank=True, db_index=True)
-    instance_type_id =  models.CharField(max_length=50,
+    instance_type_id = models.CharField(max_length=50,
-                                         null=True,
+                                        null=True,
-                                         blank=True,
+                                        blank=True,
-                                         db_index=True)
+                                        db_index=True)
     status = models.CharField(max_length=50, db_index=True,
                               choices=STATUS_CHOICES,
                               default=PENDING)

tests/unit/test_dbapi.py

@@ -20,6 +20,8 @@ class StacktachRawParsingTestCase(unittest.TestCase):
         fake_model = self.mox.CreateMockAnything()
         fake_meta = self.mox.CreateMockAnything()
         fake_model._meta = fake_meta
+        fake_orm = self.mox.CreateMockAnything()
+        fake_model.objects = fake_orm
         return fake_model
 
     def test_get_filter_args(self):
@@ -82,3 +84,118 @@ class StacktachRawParsingTestCase(unittest.TestCase):
                           fake_model, fake_request)
 
         self.mox.VerifyAll()
+
+    def test_get_db_objects(self):
+        fake_model = self.make_fake_model()
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.GET = {}
+        self.mox.StubOutWithMock(dbapi, '_get_filter_args')
+        dbapi._get_filter_args(fake_model, fake_request).AndReturn({})
+        self.mox.StubOutWithMock(dbapi, '_check_has_field')
+        dbapi._check_has_field(fake_model, 'id')
+        result = self.mox.CreateMockAnything()
+        fake_model.objects.all().AndReturn(result)
+        result.order_by('id').AndReturn(result)
+        result.__getitem__(slice(None, None, None)).AndReturn(result)
+        self.mox.ReplayAll()
+
+        query_result = dbapi.get_db_objects(fake_model, fake_request, 'id')
+        self.assertEquals(query_result, result)
+
+        self.mox.VerifyAll()
+
+    def test_get_db_objects_desc(self):
+        fake_model = self.make_fake_model()
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.GET = {'direction': 'desc'}
+        self.mox.StubOutWithMock(dbapi, '_get_filter_args')
+        dbapi._get_filter_args(fake_model, fake_request).AndReturn({})
+        self.mox.StubOutWithMock(dbapi, '_check_has_field')
+        dbapi._check_has_field(fake_model, 'id')
+        result = self.mox.CreateMockAnything()
+        fake_model.objects.all().AndReturn(result)
+        result.order_by('-id').AndReturn(result)
+        result.__getitem__(slice(None, None, None)).AndReturn(result)
+        self.mox.ReplayAll()
+
+        query_result = dbapi.get_db_objects(fake_model, fake_request, 'id')
+        self.assertEquals(query_result, result)
+
+        self.mox.VerifyAll()
+
+    def test_get_db_objects_limit(self):
+        fake_model = self.make_fake_model()
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.GET = {'limit': 1}
+        self.mox.StubOutWithMock(dbapi, '_get_filter_args')
+        dbapi._get_filter_args(fake_model, fake_request).AndReturn({})
+        self.mox.StubOutWithMock(dbapi, '_check_has_field')
+        dbapi._check_has_field(fake_model, 'id')
+        result = self.mox.CreateMockAnything()
+        fake_model.objects.all().AndReturn(result)
+        result.order_by('id').AndReturn(result)
+        result.__getitem__(slice(None, 1, None)).AndReturn(result)
+        self.mox.ReplayAll()
+
+        query_result = dbapi.get_db_objects(fake_model, fake_request, 'id')
+        self.assertEquals(query_result, result)
+
+        self.mox.VerifyAll()
+
+    def test_get_db_objects_offset(self):
+        fake_model = self.make_fake_model()
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.GET = {'offset': 1}
+        self.mox.StubOutWithMock(dbapi, '_get_filter_args')
+        dbapi._get_filter_args(fake_model, fake_request).AndReturn({})
+        self.mox.StubOutWithMock(dbapi, '_check_has_field')
+        dbapi._check_has_field(fake_model, 'id')
+        result = self.mox.CreateMockAnything()
+        fake_model.objects.all().AndReturn(result)
+        result.order_by('id').AndReturn(result)
+        result.__getitem__(slice(1, None, None)).AndReturn(result)
+        self.mox.ReplayAll()
+
+        query_result = dbapi.get_db_objects(fake_model, fake_request, 'id')
+        self.assertEquals(query_result, result)
+
+        self.mox.VerifyAll()
+
+    def test_get_db_objects_offset_and_limit(self):
+        fake_model = self.make_fake_model()
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.GET = {'offset': 2, 'limit': 2}
+        self.mox.StubOutWithMock(dbapi, '_get_filter_args')
+        dbapi._get_filter_args(fake_model, fake_request).AndReturn({})
+        self.mox.StubOutWithMock(dbapi, '_check_has_field')
+        dbapi._check_has_field(fake_model, 'id')
+        result = self.mox.CreateMockAnything()
+        fake_model.objects.all().AndReturn(result)
+        result.order_by('id').AndReturn(result)
+        result.__getslice__(2, 4).AndReturn(result)
+        self.mox.ReplayAll()
+
+        query_result = dbapi.get_db_objects(fake_model, fake_request, 'id')
+        self.assertEquals(query_result, result)
+
+        self.mox.VerifyAll()
+
+    def test_get_db_objects_with_filter(self):
+        fake_model = self.make_fake_model()
+        fake_request = self.mox.CreateMockAnything()
+        filters = {'instance': INSTANCE_ID_1}
+        fake_request.GET = filters
+        self.mox.StubOutWithMock(dbapi, '_get_filter_args')
+        dbapi._get_filter_args(fake_model, fake_request).AndReturn(filters)
+        self.mox.StubOutWithMock(dbapi, '_check_has_field')
+        dbapi._check_has_field(fake_model, 'id')
+        result = self.mox.CreateMockAnything()
+        fake_model.objects.filter(**filters).AndReturn(result)
+        result.order_by('id').AndReturn(result)
+        result.__getitem__(slice(None, None, None)).AndReturn(result)
+        self.mox.ReplayAll()
+
+        query_result = dbapi.get_db_objects(fake_model, fake_request, 'id')
+        self.assertEquals(query_result, result)
+
+        self.mox.VerifyAll()

tests/unit/test_worker.py

@@ -9,6 +9,7 @@ import mox
 from stacktach import db, views
 import worker.worker as worker
 
+
 class NovaConsumerTestCase(unittest.TestCase):
     def setUp(self):
         self.mox = mox.Mox()