diff --git a/docs/api.rst b/docs/api.rst index 8b559dc..0644414 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -805,4 +805,4 @@ stacky/usage/exists ] ] - :query instance: desired instance UUID (optional) + :query instance: desired instance UUID (optional) \ No newline at end of file diff --git a/docs/dbapi.rst b/docs/dbapi.rst index d23cac4..954caaa 100644 --- a/docs/dbapi.rst +++ b/docs/dbapi.rst @@ -874,4 +874,75 @@ Returns a single instance exists matching provided id "id": 5300, "delete": null } - } \ No newline at end of file + } + +db/count/verified/ +================== + +.. http:get:: http://example.com/count/verified/ + +Returns a count of .verified events stored in Stacktach's Rawdata table from +``audit_period_beginning`` to ``audit_period_ending`` + + **Query Parameters** + + * ``audit_period_beginning``: datetime (yyyy-mm-dd) + * ``audit_period_ending``: datetime (yyyy-mm-dd) + * ``service``: ``nova`` or ``glance``. default="nova" + + **Example request**: + + .. sourcecode:: http + + GET db/count/verified/ HTTP/1.1 + Host: example.com + Accept: application/json + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: application/json + + { + count: 10 + } + +repair +====== + +.. http:post:: http://example.com/repair/ + + Changes the status of all the exists of message-ids sent with the request + from 'pending' to 'sent_unverified' so that the verifier does not end up + sending .verified for all those exists(since the .exists have already been + modified as .verified and sent to AH by Yagi). It sends back the message-ids + of exists which could not be updated in the json response. + + **Example request**: + + .. sourcecode::http + + POST /repair/ HTTP/1.1 + Host: example.com + Accept: application/json + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/json + + { + u'exists_not_pending': [u'494ebfce-0219-4b62-b810-79039a279620'], + u'absent_exists': [u'7609f3b2-3694-4b6f-869e-2f13ae504cb2', + u'0c64032e-4a60-44c0-a99d-5a4f2e46afb0'] + } + + :query message_ids: list of message_ids of exists messages + :query service: ``nova`` or ``glance``. default="nova" diff --git a/stacktach/dbapi.py b/stacktach/dbapi.py index 174c50b..f170f34 100644 --- a/stacktach/dbapi.py +++ b/stacktach/dbapi.py @@ -21,6 +21,7 @@ import decimal import functools import json +from datetime import datetime from django.db import transaction from django.db.models import Count @@ -445,3 +446,50 @@ def _convert_model_list(model_list, extra_values_func=None): converted.append(_convert_model(item, extra_values_func)) return converted + + +def _rawdata_factory(service): + if service == "nova": + rawdata = models.RawData.objects + elif service == "glance": + rawdata = models.GlanceRawData.objects + else: + raise BadRequestException(message="Invalid service") + return rawdata + + +@api_call +def get_verified_count(request): + try: + audit_period_beginning = datetime.strptime( + request.GET.get("audit_period_beginning"), "%Y-%m-%d") + audit_period_ending = datetime.strptime( + request.GET.get("audit_period_ending"), "%Y-%m-%d") + service = request.GET.get("service", "nova") + rawdata = _rawdata_factory(service) + filters = { + 'when__gte': dt.dt_to_decimal(audit_period_beginning), + 'when__lte': dt.dt_to_decimal(audit_period_ending), + 'event': "compute.instance.exists.verified" + } + return {'count': rawdata.filter(**filters).count()} + except KeyError and TypeError: + raise BadRequestException(message="Invalid/absent query parameter") + except ValueError: + raise BadRequestException(message="Invalid format for date (Correct " + "format should be %YYYY-%mm-%dd)") + + +def repair_stacktach_down(request): + post_dict = dict((request.POST._iterlists())) + message_ids = post_dict.get('message_ids') + service = post_dict.get('service', ['nova']) + klass = _exists_model_factory(service[0])['klass'] + absent_exists, exists_not_pending = \ + klass.mark_exists_as_sent_unverified(message_ids) + response_data = {'absent_exists': absent_exists, + 'exists_not_pending': exists_not_pending} + response = HttpResponse(json.dumps(response_data), + content_type="application/json") + return response + diff --git a/stacktach/models.py b/stacktach/models.py index 9d1e0fb..d9afe35 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -338,6 +338,24 @@ class InstanceExists(models.Model): def update_status(self, new_status): self.status = new_status + @staticmethod + def mark_exists_as_sent_unverified(message_ids): + absent_exists = [] + exists_not_pending = [] + for message_id in message_ids: + try: + exists = InstanceExists.objects.get(message_id=message_id) + if exists.status == InstanceExists.PENDING: + exists.status = InstanceExists.SENT_UNVERIFIED + exists.save() + else: + exists_not_pending.append(message_id) + except Exception: + absent_exists.append(message_id) + return absent_exists, exists_not_pending + + + class Timing(models.Model): """Each Timing record corresponds to a .start/.end event pair @@ -536,6 +554,24 @@ class ImageExists(models.Model): self.fail_reason = reason self.save() + @staticmethod + def mark_exists_as_sent_unverified(message_ids): + absent_exists = [] + exists_not_pending = [] + for message_id in message_ids: + exists_list = ImageExists.objects.filter(message_id=message_id) + if exists_list: + for exists in exists_list: + if exists.status == ImageExists.PENDING: + exists.status = ImageExists.SENT_UNVERIFIED + exists.save() + else: + exists_not_pending.append(message_id) + else : + absent_exists.append(message_id) + return absent_exists, exists_not_pending + + def get_model_fields(model): return model._meta.fields diff --git a/stacktach/urls.py b/stacktach/urls.py index b5d471c..2bd64de 100644 --- a/stacktach/urls.py +++ b/stacktach/urls.py @@ -91,6 +91,8 @@ dbapi_urls = ( 'stacktach.dbapi.get_usage_exist_stats'), url(r'db/stats/glance/exists$', 'stacktach.dbapi.get_usage_exist_stats_glance'), + url(r'db/count/verified', 'stacktach.dbapi.get_verified_count'), + url(r'db/repair/', 'stacktach.dbapi.repair_stacktach_down'), ) urlpatterns = patterns('', *(web_urls + stacky_urls + dbapi_urls)) diff --git a/tests/unit/test_dbapi.py b/tests/unit/test_dbapi.py index 3de9b11..07d1b84 100644 --- a/tests/unit/test_dbapi.py +++ b/tests/unit/test_dbapi.py @@ -19,6 +19,7 @@ # IN THE SOFTWARE. import datetime +from decimal import Decimal import json from django.db.models import Count @@ -129,8 +130,8 @@ class DBAPITestCase(StacktachBaseTestCase): fake_request = self.mox.CreateMockAnything() fake_request.GET = {'somebadfield_max': str(start_time)} fake_model = self.make_fake_model() - fake_model._meta.get_field_by_name('somebadfield')\ - .AndRaise(FieldDoesNotExist()) + fake_model._meta.get_field_by_name('somebadfield') \ + .AndRaise(FieldDoesNotExist()) self.mox.ReplayAll() self.assertRaises(dbapi.BadRequestException, dbapi._get_filter_args, @@ -312,7 +313,8 @@ class DBAPITestCase(StacktachBaseTestCase): fake_request.GET = filters self.mox.StubOutWithMock(dbapi, '_get_filter_args') dbapi._get_filter_args(fake_model, fake_request, - custom_filters=custom_filters).AndReturn(filters) + custom_filters=custom_filters).AndReturn( + filters) self.mox.StubOutWithMock(dbapi, '_check_has_field') dbapi._check_has_field(fake_model, 'id') result = self.mox.CreateMockAnything() @@ -563,7 +565,8 @@ class DBAPITestCase(StacktachBaseTestCase): exists1.send_status = 200 self.mox.VerifyAll() - def test_send_status_batch_accepts_post_for_nova_and_glance_when_version_is_1(self): + def test_send_status_batch_accepts_post_for_nova_and_glance_when_version_is_1( + self): fake_request = self.mox.CreateMockAnything() fake_request.method = 'POST' fake_request.GET = {'service': 'glance'} @@ -591,14 +594,16 @@ class DBAPITestCase(StacktachBaseTestCase): models.ImageExists.objects.select_for_update().AndReturn(results1) exists1A = self.mox.CreateMockAnything() exists1B = self.mox.CreateMockAnything() - results1.filter(message_id=MESSAGE_ID_2).AndReturn([exists1A, exists1B]) + results1.filter(message_id=MESSAGE_ID_2).AndReturn( + [exists1A, exists1B]) exists1A.save() exists1B.save() results2 = self.mox.CreateMockAnything() models.ImageExists.objects.select_for_update().AndReturn(results2) exists2A = self.mox.CreateMockAnything() exists2B = self.mox.CreateMockAnything() - results2.filter(message_id=MESSAGE_ID_1).AndReturn([exists2A, exists2B]) + results2.filter(message_id=MESSAGE_ID_1).AndReturn( + [exists2A, exists2B]) exists2A.save() exists2B.save() trans_obj.__exit__(None, None, None) @@ -609,7 +614,6 @@ class DBAPITestCase(StacktachBaseTestCase): self.mox.VerifyAll() - def test_send_status_batch_accepts_post_when_version_is_0(self): fake_request = self.mox.CreateMockAnything() fake_request.method = 'POST' @@ -764,7 +768,8 @@ class DBAPITestCase(StacktachBaseTestCase): launches = {'a': 1} self.mox.StubOutWithMock(dbapi, '_convert_model_list') dbapi._convert_model_list(mock_objects).AndReturn(launches) - dbapi.get_db_objects(models.InstanceUsage, fake_request, 'launched_at').AndReturn(mock_objects) + dbapi.get_db_objects(models.InstanceUsage, fake_request, + 'launched_at').AndReturn(mock_objects) self.mox.ReplayAll() resp = dbapi.list_usage_launches(fake_request) @@ -781,7 +786,8 @@ class DBAPITestCase(StacktachBaseTestCase): launches = {'a': 1} self.mox.StubOutWithMock(dbapi, '_convert_model_list') dbapi._convert_model_list(mock_objects).AndReturn(launches) - dbapi.get_db_objects(models.ImageUsage, fake_request, 'created_at').AndReturn(mock_objects) + dbapi.get_db_objects(models.ImageUsage, fake_request, + 'created_at').AndReturn(mock_objects) self.mox.ReplayAll() resp = dbapi.list_usage_images(fake_request) @@ -798,7 +804,8 @@ class DBAPITestCase(StacktachBaseTestCase): launches = {'a': 1} self.mox.StubOutWithMock(dbapi, '_convert_model_list') dbapi._convert_model_list(mock_objects).AndReturn(launches) - dbapi.get_db_objects(models.InstanceUsage, fake_request, 'launched_at').AndReturn(mock_objects) + dbapi.get_db_objects(models.InstanceUsage, fake_request, + 'launched_at').AndReturn(mock_objects) self.mox.ReplayAll() resp = dbapi.list_usage_launches(fake_request) @@ -885,7 +892,8 @@ class DBAPITestCase(StacktachBaseTestCase): deletes = {'a': 1} self.mox.StubOutWithMock(dbapi, '_convert_model_list') dbapi._convert_model_list(mock_objects).AndReturn(deletes) - dbapi.get_db_objects(models.InstanceDeletes, fake_request, 'launched_at').AndReturn(mock_objects) + dbapi.get_db_objects(models.InstanceDeletes, fake_request, + 'launched_at').AndReturn(mock_objects) self.mox.ReplayAll() resp = dbapi.list_usage_deletes(fake_request) @@ -902,7 +910,8 @@ class DBAPITestCase(StacktachBaseTestCase): deletes = {'a': 1} self.mox.StubOutWithMock(dbapi, '_convert_model_list') dbapi._convert_model_list(mock_objects).AndReturn(deletes) - dbapi.get_db_objects(models.InstanceDeletes, fake_request, 'launched_at').AndReturn(mock_objects) + dbapi.get_db_objects(models.InstanceDeletes, fake_request, + 'launched_at').AndReturn(mock_objects) self.mox.ReplayAll() resp = dbapi.list_usage_deletes(fake_request) @@ -919,7 +928,8 @@ class DBAPITestCase(StacktachBaseTestCase): deletes = {'a': 1} self.mox.StubOutWithMock(dbapi, '_convert_model_list') dbapi._convert_model_list(mock_objects).AndReturn(deletes) - dbapi.get_db_objects(models.ImageDeletes, fake_request, 'deleted_at').AndReturn(mock_objects) + dbapi.get_db_objects(models.ImageDeletes, fake_request, + 'deleted_at').AndReturn(mock_objects) self.mox.ReplayAll() resp = dbapi.list_usage_deletes_glance(fake_request) @@ -1092,3 +1102,118 @@ class DBAPITestCase(StacktachBaseTestCase): expected_response = json.dumps({'stats': result}) self.assertEqual(expected_response, response.content) self.mox.VerifyAll() + + def test_get_verified_count(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + fake_request.GET = {'audit_period_beginning': "2014-02-26", + 'audit_period_ending': "2014-02-27", + 'service': "nova"} + mock_query = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models.RawData.objects, "filter") + models.RawData.objects.filter(event='compute.instance.exists.verified', + when__gte=Decimal('1393372800'), + when__lte=Decimal('1393459200')).\ + AndReturn(mock_query) + mock_query.count().AndReturn(100) + self.mox.ReplayAll() + + response = dbapi.get_verified_count(fake_request) + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.content), {'count': 100}) + self.mox.VerifyAll() + + def test_get_verified_count_wrong_date_format_returns_400(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + fake_request.GET = {'audit_period_beginning': "2014-020-26", + + 'service': "nova"} + + self.mox.ReplayAll() + + response = dbapi.get_verified_count(fake_request) + self.assertEqual(response.status_code, 400) + self.assertEqual(json.loads(response.content)['message'], + "Invalid format for date" + " (Correct format should be %YYYY-%mm-%dd)") + self.mox.VerifyAll() + + def test_get_verified_count_wrong_service_returns_400(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + fake_request.GET = {'audit_period_beginning': "2014-02-26", + "audit_period_ending": "2014-02-27", + 'service': "qonos"} + + self.mox.ReplayAll() + + response = dbapi.get_verified_count(fake_request) + self.assertEqual(response.status_code, 400) + self.assertEqual(json.loads(response.content)['message'], + "Invalid service") + self.mox.VerifyAll() + + def test_get_verified_count_invalid_query_parameter_returns_400(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + fake_request.GET = {'audit_period': "2014-02-26",} + + self.mox.ReplayAll() + + response = dbapi.get_verified_count(fake_request) + self.assertEqual(response.status_code, 400) + self.assertEqual(json.loads(response.content)['message'], + "Invalid/absent query parameter") + self.mox.VerifyAll() + +class StacktachRepairScenarioApi(StacktachBaseTestCase): + def setUp(self): + self.mox = mox.Mox() + + def tearDown(self): + self.mox.UnsetStubs() + + def test_change_nova_exists_status_for_all_exists(self): + request = self.mox.CreateMockAnything() + request.POST = self.mox.CreateMockAnything() + message_ids = ["04fd94b5-64dd-4559-83b7-981d9d4f7a5a", + "14fd94b5-64dd-4559-83b7-981d9d4f7a5a", + "24fd94b5-64dd-4559-83b7-981d9d4f7a5a"] + request.POST._iterlists().AndReturn([('service', ['nova']), + ('message_ids', message_ids)]) + self.mox.StubOutWithMock(models.InstanceExists, + 'mark_exists_as_sent_unverified') + models.InstanceExists.mark_exists_as_sent_unverified(message_ids).\ + AndReturn([[], []]) + self.mox.ReplayAll() + + response = dbapi.repair_stacktach_down(request) + self.assertEqual(response.status_code, 200) + response_data = json.loads(response.content) + self.assertEqual(response_data['exists_not_pending'], []) + self.assertEqual(response_data['absent_exists'], []) + + self.mox.VerifyAll() + + def test_change_glance_exists_status_for_all_exists(self): + request = self.mox.CreateMockAnything() + request.POST = self.mox.CreateMockAnything() + message_ids = ['04fd94b5-64dd-4559-83b7-981d9d4f7a5a', + '14fd94b5-64dd-4559-83b7-981d9d4f7a5a', + '24fd94b5-64dd-4559-83b7-981d9d4f7a5a'] + request.POST._iterlists().AndReturn([('service', ['glance']), + ('message_ids', message_ids)]) + self.mox.StubOutWithMock(models.ImageExists, + 'mark_exists_as_sent_unverified') + models.ImageExists.mark_exists_as_sent_unverified(message_ids).\ + AndReturn([[], []]) + self.mox.ReplayAll() + + response = dbapi.repair_stacktach_down(request) + self.assertEqual(response.status_code, 200) + response_data = json.loads(response.content) + self.assertEqual(response_data['exists_not_pending'], []) + self.assertEqual(response_data['absent_exists'], []) + + self.mox.VerifyAll() diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index ea32261..f2c7a03 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -112,6 +112,81 @@ class ImageExistsTestCase(unittest.TestCase): 'owner1-3': [exist4], 'owner2-2': [exist2]}) + def test_mark_exists_as_sent_unverified(self): + message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b", + "9156b83e-f684-4ec3-8f94-7e41902f27aa"] + + exist1 = self.mox.CreateMockAnything() + exist1.status = "pending" + exist1.save() + exist2 = self.mox.CreateMockAnything() + exist2.status = "pending" + exist2.save() + exist3 = self.mox.CreateMockAnything() + exist3.status = "pending" + exist3.save() + self.mox.StubOutWithMock(ImageExists.objects, 'filter') + ImageExists.objects.filter(message_id=message_ids[0]).AndReturn( + [exist1, exist2]) + ImageExists.objects.filter(message_id=message_ids[1]).AndReturn( + [exist3]) + self.mox.ReplayAll() + + results = ImageExists.mark_exists_as_sent_unverified(message_ids) + + self.assertEqual(results, ([], [])) + + self.mox.VerifyAll() + + def test_mark_exists_as_sent_unverified_return_absent_exists(self): + message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b", + "9156b83e-f684-4ec3-8f94-7e41902f27aa"] + + exist1 = self.mox.CreateMockAnything() + exist1.status = "pending" + exist1.save() + exist2 = self.mox.CreateMockAnything() + exist2.status = "pending" + exist2.save() + self.mox.StubOutWithMock(ImageExists.objects, 'filter') + ImageExists.objects.filter(message_id=message_ids[0]).AndReturn( + [exist1, exist2]) + ImageExists.objects.filter(message_id=message_ids[1]).AndReturn([]) + self.mox.ReplayAll() + + results = ImageExists.mark_exists_as_sent_unverified(message_ids) + + self.assertEqual(results, (['9156b83e-f684-4ec3-8f94-7e41902f27aa'], + [])) + + self.mox.VerifyAll() + + def test_mark_exists_as_sent_unverified_and_return_exist_not_pending(self): + message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b", + "9156b83e-f684-4ec3-8f94-7e41902f27aa"] + + exist1 = self.mox.CreateMockAnything() + exist1.status = "pending" + exist1.save() + exist2 = self.mox.CreateMockAnything() + exist2.status = "verified" + exist3 = self.mox.CreateMockAnything() + exist3.status = "pending" + exist3.save() + self.mox.StubOutWithMock(ImageExists.objects, 'filter') + ImageExists.objects.filter(message_id=message_ids[0]).AndReturn( + [exist1, exist2]) + ImageExists.objects.filter(message_id=message_ids[1]).AndReturn( + [exist3]) + self.mox.ReplayAll() + + results = ImageExists.mark_exists_as_sent_unverified(message_ids) + + self.assertEqual(results, ([], + ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b"])) + + self.mox.VerifyAll() + class InstanceExistsTestCase(unittest.TestCase): def setUp(self): @@ -137,3 +212,66 @@ class InstanceExistsTestCase(unittest.TestCase): self.mox.VerifyAll() self.assertEqual(results, [1, 2]) + + def test_mark_exists_as_sent_unverified(self): + message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b", + "9156b83e-f684-4ec3-8f94-7e41902f27aa"] + + exist1 = self.mox.CreateMockAnything() + exist1.status = "pending" + exist1.save() + exist2 = self.mox.CreateMockAnything() + exist2.status = "pending" + exist2.save() + self.mox.StubOutWithMock(InstanceExists.objects, 'get') + InstanceExists.objects.get(message_id=message_ids[0]).AndReturn(exist1) + InstanceExists.objects.get(message_id=message_ids[1]).AndReturn(exist2) + self.mox.ReplayAll() + + results = InstanceExists.mark_exists_as_sent_unverified(message_ids) + + self.assertEqual(results, ([], [])) + + self.mox.VerifyAll() + + def test_mark_exists_as_sent_unverified_return_absent_exists(self): + message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b", + "9156b83e-f684-4ec3-8f94-7e41902f27aa"] + + exist1 = self.mox.CreateMockAnything() + exist1.status = "pending" + exist1.save() + self.mox.StubOutWithMock(InstanceExists.objects, 'get') + InstanceExists.objects.get(message_id=message_ids[0]).AndReturn(exist1) + InstanceExists.objects.get(message_id=message_ids[1]).AndRaise( + Exception) + self.mox.ReplayAll() + + results = InstanceExists.mark_exists_as_sent_unverified(message_ids) + + self.assertEqual(results, (['9156b83e-f684-4ec3-8f94-7e41902f27aa'], + [])) + + self.mox.VerifyAll() + + def test_mark_exists_as_sent_unverified_and_return_exist_not_pending(self): + message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b", + "9156b83e-f684-4ec3-8f94-7e41902f27aa"] + + exist1 = self.mox.CreateMockAnything() + exist1.status = "pending" + exist1.save() + exist2 = self.mox.CreateMockAnything() + exist2.status = "verified" + self.mox.StubOutWithMock(InstanceExists.objects, 'get') + InstanceExists.objects.get(message_id=message_ids[0]).AndReturn(exist1) + InstanceExists.objects.get(message_id=message_ids[1]).AndReturn(exist2) + self.mox.ReplayAll() + + results = InstanceExists.mark_exists_as_sent_unverified(message_ids) + + self.assertEqual(results, ([], + ["9156b83e-f684-4ec3-8f94-7e41902f27aa"])) + + self.mox.VerifyAll() + diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py index 4ca4311..a668f13 100644 --- a/tests/unit/test_stacktach.py +++ b/tests/unit/test_stacktach.py @@ -38,8 +38,7 @@ from utils import TENANT_ID_1 from utils import INSTANCE_TYPE_ID_1 from utils import DUMMY_TIME from utils import INSTANCE_TYPE_ID_2 -from utils import IMAGE_UUID_1 -from stacktach import stacklog +from stacktach import stacklog, models from stacktach import notification from stacktach import views from tests.unit import StacktachBaseTestCase