From fef0458472d6e1e00931fd8c2df831a96eb38f5c Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 3 Mar 2014 12:27:49 -0500 Subject: [PATCH 1/3] More efficient event stats query --- stacktach/dbapi.py | 18 +++++++--- tests/unit/test_dbapi.py | 74 +++++++++++++++++++++++++++++++++------- 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/stacktach/dbapi.py b/stacktach/dbapi.py index a593d78..0f98ebc 100644 --- a/stacktach/dbapi.py +++ b/stacktach/dbapi.py @@ -470,12 +470,22 @@ def get_event_stats(request): when_max = utils.str_time_to_unix(request.GET['when_max']) filters['when__lte'] = when_max - if 'event' in request.GET: - filters['event'] = request.GET['event'] - service = request.GET.get("service", "nova") rawdata = _rawdata_factory(service) - return {'stats': {'count': rawdata.filter(**filters).count()}} + if filters: + rawdata = rawdata.filter(**filters) + events = rawdata.values('event').annotate(event_count=Count('event')) + events = list(events) + + if 'event' in request.GET: + event_filter = request.GET['event'] + event_count = {'event': event_filter, 'event_count': 0} + for event in events: + if event['event'] == event_filter: + event_count['event_count'] = event['event_count'] + events = [event_count, ] + + return {'stats': events} except (KeyError, TypeError): raise BadRequestException(message="Invalid/absent query parameter") except (ValueError, AttributeError): diff --git a/tests/unit/test_dbapi.py b/tests/unit/test_dbapi.py index b7e66f7..ba93418 100644 --- a/tests/unit/test_dbapi.py +++ b/tests/unit/test_dbapi.py @@ -43,10 +43,13 @@ class DBAPITestCase(StacktachBaseTestCase): self.mox = mox.Mox() dne_exception = models.InstanceExists.DoesNotExist mor_exception = models.InstanceExists.MultipleObjectsReturned + self.mox.StubOutWithMock(models, 'RawData', + use_mock_anything=True) self.mox.StubOutWithMock(models, 'InstanceExists', use_mock_anything=True) self.mox.StubOutWithMock(models, 'ImageExists', use_mock_anything=True) + models.RawData.objects = self.mox.CreateMockAnything() models.InstanceExists._meta = self.mox.CreateMockAnything() models.ImageExists._meta = self.mox.CreateMockAnything() models.InstanceExists.objects = self.mox.CreateMockAnything() @@ -1103,26 +1106,71 @@ class DBAPITestCase(StacktachBaseTestCase): self.assertEqual(expected_response, response.content) self.mox.VerifyAll() - def test_get_verified_count(self): + def test_get_event_stats(self): fake_request = self.mox.CreateMockAnything() fake_request.method = 'GET' - fake_request.GET = {'when_min': "2014-02-26 00:00:00", - 'when_max': "2014-02-27 00:00:00", - 'service': "nova", - 'event': 'compute.instance.exists.verified'} + fake_request.GET = {'service': "nova"} mock_query = self.mox.CreateMockAnything() - self.mox.StubOutWithMock(models.RawData.objects, "filter") - models.RawData.objects.filter(event='compute.instance.exists.verified', - when__gte=Decimal('1393372800'), - when__lte=Decimal('1393459200')).\ - AndReturn(mock_query) - mock_query.count().AndReturn(100) + models.RawData.objects.values('event').AndReturn(mock_query) + events = [ + {'event': 'compute.instance.exists.verified', 'event_count': 100}, + {'event': 'compute.instance.exists', 'event_count': 100} + ] + mock_query.annotate(event_count=mox.IsA(Count)).AndReturn(events) self.mox.ReplayAll() response = dbapi.get_event_stats(fake_request) self.assertEqual(response.status_code, 200) - self.assertEqual(json.loads(response.content), - {'stats': {'count': 100}}) + self.assertEqual(response.content, + json.dumps({'stats': events})) + self.mox.VerifyAll() + + def test_get_event_stats_date_range(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + start = "2014-02-26 00:00:00" + end = "2014-02-27 00:00:00" + fake_request.GET = {'when_min': start, + 'when_max': end, + 'service': "nova"} + mock_query = self.mox.CreateMockAnything() + filters = { + 'when__gte': stacktach_utils.str_time_to_unix(start), + 'when__lte': stacktach_utils.str_time_to_unix(end) + } + models.RawData.objects.filter(**filters).AndReturn(mock_query) + mock_query.values('event').AndReturn(mock_query) + events = [ + {'event': 'compute.instance.exists.verified', 'event_count': 100}, + {'event': 'compute.instance.exists', 'event_count': 100} + ] + mock_query.annotate(event_count=mox.IsA(Count)).AndReturn(events) + self.mox.ReplayAll() + + response = dbapi.get_event_stats(fake_request) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, + json.dumps({'stats': events})) + self.mox.VerifyAll() + + def test_get_verified_count(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + fake_request.GET = {'service': "nova", + 'event': 'compute.instance.exists.verified'} + mock_query = self.mox.CreateMockAnything() + models.RawData.objects.values('event').AndReturn(mock_query) + events = [ + {'event': 'compute.instance.exists.verified', 'event_count': 100}, + {'event': 'compute.instance.exists', 'event_count': 100} + ] + mock_query.annotate(event_count=mox.IsA(Count)).AndReturn(events) + self.mox.ReplayAll() + + response = dbapi.get_event_stats(fake_request) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, + json.dumps({'stats': [events[0]]})) self.mox.VerifyAll() def test_get_verified_count_wrong_date_format_returns_400(self): From 18c3309d7baaa2f23bfa24b20032b58a76480d6b Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 3 Mar 2014 12:52:42 -0500 Subject: [PATCH 2/3] Short circuit event stats filter when event found --- stacktach/dbapi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stacktach/dbapi.py b/stacktach/dbapi.py index 0f98ebc..b9f7528 100644 --- a/stacktach/dbapi.py +++ b/stacktach/dbapi.py @@ -483,6 +483,7 @@ def get_event_stats(request): for event in events: if event['event'] == event_filter: event_count['event_count'] = event['event_count'] + break events = [event_count, ] return {'stats': events} From e21afc84a76c62231b3a770496c408de055ea186 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 3 Mar 2014 13:06:02 -0500 Subject: [PATCH 3/3] Clearer event stats filtering --- stacktach/dbapi.py | 10 +++------- tests/unit/test_dbapi.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/stacktach/dbapi.py b/stacktach/dbapi.py index b9f7528..55b795b 100644 --- a/stacktach/dbapi.py +++ b/stacktach/dbapi.py @@ -478,13 +478,9 @@ def get_event_stats(request): events = list(events) if 'event' in request.GET: - event_filter = request.GET['event'] - event_count = {'event': event_filter, 'event_count': 0} - for event in events: - if event['event'] == event_filter: - event_count['event_count'] = event['event_count'] - break - events = [event_count, ] + event = request.GET['event'] + default = {'event': event, 'event_count': 0} + events = [x for x in events if x['event'] == event] or [default, ] return {'stats': events} except (KeyError, TypeError): diff --git a/tests/unit/test_dbapi.py b/tests/unit/test_dbapi.py index ba93418..3ebfaeb 100644 --- a/tests/unit/test_dbapi.py +++ b/tests/unit/test_dbapi.py @@ -1173,6 +1173,26 @@ class DBAPITestCase(StacktachBaseTestCase): json.dumps({'stats': [events[0]]})) self.mox.VerifyAll() + def test_get_verified_count_default(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + fake_request.GET = {'service': "nova", + 'event': 'compute.instance.exists.verified'} + mock_query = self.mox.CreateMockAnything() + models.RawData.objects.values('event').AndReturn(mock_query) + events = [ + {'event': 'compute.instance.create.start', 'event_count': 100}, + {'event': 'compute.instance.exists', 'event_count': 100} + ] + mock_query.annotate(event_count=mox.IsA(Count)).AndReturn(events) + self.mox.ReplayAll() + + response = dbapi.get_event_stats(fake_request) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, + json.dumps({'stats': [{'event': 'compute.instance.exists.verified', 'event_count': 0}]})) + self.mox.VerifyAll() + def test_get_verified_count_wrong_date_format_returns_400(self): fake_request = self.mox.CreateMockAnything() fake_request.method = 'GET'