From 8d11342c396398f4efa970d7e97ce3ee4bdd370d Mon Sep 17 00:00:00 2001 From: Anuj Mathur Date: Tue, 3 Sep 2013 15:57:56 +0530 Subject: [PATCH] Modified glance worker to process the new exists notification --- stacktach/notification.py | 73 +++--- tests/unit/test_notification.py | 409 ++++++++++++++++++++------------ 2 files changed, 295 insertions(+), 187 deletions(-) diff --git a/stacktach/notification.py b/stacktach/notification.py index 292aece..f10ff31 100644 --- a/stacktach/notification.py +++ b/stacktach/notification.py @@ -103,14 +103,6 @@ class GlanceNotification(Notification): self.size = self.payload.get('size', None) created_at = self.payload.get('created_at', None) self.created_at = created_at and utils.str_time_to_unix(created_at) - audit_period_beginning = self.payload.get( - 'audit_period_beginning', None) - self.audit_period_beginning = audit_period_beginning and\ - utils.str_time_to_unix(audit_period_beginning) - audit_period_ending = self.payload.get( - 'audit_period_ending', None) - self.audit_period_ending = audit_period_ending and \ - utils.str_time_to_unix(audit_period_ending) else: self.properties = {} self.image_type = None @@ -118,8 +110,6 @@ class GlanceNotification(Notification): self.uuid = None self.size = None self.created_at = None - self.audit_period_beginning = None - self.audit_period_ending = None @property def owner(self): @@ -131,6 +121,7 @@ class GlanceNotification(Notification): @property def instance(self): return self.properties.get('instance_uuid', None) + @property def deleted_at(self): deleted_at = self.body.get('deleted_at', None) @@ -157,27 +148,49 @@ class GlanceNotification(Notification): uuid=self.uuid) def save_exists(self, raw): - if self.created_at: - values = { - 'uuid': self.uuid, - 'audit_period_beginning': self.audit_period_beginning, - 'audit_period_ending': self.audit_period_ending, - 'owner': self.owner, - 'size': self.size, - 'raw': raw - } - usage = db.get_image_usage(uuid=self.uuid) - values['usage'] = usage - values['created_at'] = self.created_at - if self.deleted_at: - delete = db.get_image_delete(uuid=self.uuid) - values['delete'] = delete - values['deleted_at'] = self.deleted_at - - db.create_image_exists(**values) + if isinstance(self.payload, dict): + audit_period_beginning = self.payload.get( + 'audit_period_beginning', None) + audit_period_beginning = audit_period_beginning and\ + utils.str_time_to_unix(audit_period_beginning) + audit_period_ending = self.payload.get( + 'audit_period_ending', None) + audit_period_ending = audit_period_ending and \ + utils.str_time_to_unix(audit_period_ending) + images = self.payload.get('images', []) else: - stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)" - % raw.id) + audit_period_beginning = None + audit_period_ending = None + images = [] + + for image in images: + created_at = image['created_at'] + created_at = created_at and utils.str_time_to_unix(created_at) + uuid = image['id'] + deleted_at = image['deleted_at'] + deleted_at = deleted_at and utils.str_time_to_unix(deleted_at) + if created_at: + + values = { + 'uuid': uuid, + 'audit_period_beginning': audit_period_beginning, + 'audit_period_ending': audit_period_ending, + 'owner': self.owner, + 'size': image['size'], + 'raw': raw + } + usage = db.get_image_usage(uuid=uuid) + values['usage'] = usage + values['created_at'] = created_at + if deleted_at: + delete = db.get_image_delete(uuid=uuid) + values['delete'] = delete + values['deleted_at'] = deleted_at + + db.create_image_exists(**values) + else: + stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)" + % raw.id) def save_usage(self, raw): values = { diff --git a/tests/unit/test_notification.py b/tests/unit/test_notification.py index 5c8a426..1beb02e 100644 --- a/tests/unit/test_notification.py +++ b/tests/unit/test_notification.py @@ -22,7 +22,7 @@ import json import mox -from stacktach import notification +from stacktach import notification, stacklog from stacktach import utils from stacktach.notification import Notification @@ -316,162 +316,6 @@ class GlanceNotificationTestCase(StacktachBaseTestCase): self.assertEquals(notification.save(), raw) self.mox.VerifyAll() - def test_save_image_exists(self): - raw = self.mox.CreateMockAnything() - audit_period_beginning = "2013-05-20 17:31:57.939614" - audit_period_ending = "2013-06-20 17:31:57.939614" - size = 123 - uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" - body = { - "event_type": "image.upload", - "timestamp": "2013-06-20 18:31:57.939614", - "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", - "payload": { - "created_at": str(DUMMY_TIME), - "status": "saving", - "audit_period_beginning": audit_period_beginning, - "audit_period_ending": audit_period_ending, - "properties": { - "image_type": "snapshot", - "instance_uuid": INSTANCE_ID_1, - }, - "size": size, - "owner": TENANT_ID_1, - "id": uuid - } - } - deployment = "1" - routing_key = "glance_monitor.info" - json_body = json.dumps([routing_key, body]) - - self.mox.StubOutWithMock(db, 'create_image_exists') - self.mox.StubOutWithMock(db, 'get_image_usage') - - db.get_image_usage(uuid=uuid).AndReturn(None) - db.create_image_exists( - created_at=utils.str_time_to_unix(str(DUMMY_TIME)), - owner=TENANT_ID_1, - raw=raw, - audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), - audit_period_ending=utils.str_time_to_unix(audit_period_ending), - size=size, - uuid=uuid, - usage=None).AndReturn(raw) - - self.mox.ReplayAll() - - notification = GlanceNotification(body, deployment, routing_key, - json_body) - notification.save_exists(raw) - self.mox.VerifyAll() - - def test_save_image_exists_with_delete_not_none(self): - raw = self.mox.CreateMockAnything() - delete = self.mox.CreateMockAnything() - audit_period_beginning = "2013-05-20 17:31:57.939614" - audit_period_ending = "2013-06-20 17:31:57.939614" - size = 123 - uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" - deleted_at = "2013-06-20 14:31:57.939614" - body = { - "event_type": "image.upload", - "timestamp": "2013-06-20 18:31:57.939614", - "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", - "payload": { - "created_at": str(DUMMY_TIME), - "status": "saving", - "audit_period_beginning": audit_period_beginning, - "audit_period_ending": audit_period_ending, - "properties": { - "image_type": "snapshot", - "instance_uuid": INSTANCE_ID_1, - }, - "deleted_at": deleted_at, - "size": size, - "owner": TENANT_ID_1, - "id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb", - } - } - deployment = "1" - routing_key = "glance_monitor.info" - json_body = json.dumps([routing_key, body]) - - self.mox.StubOutWithMock(db, 'create_image_exists') - self.mox.StubOutWithMock(db, 'get_image_usage') - self.mox.StubOutWithMock(db, 'get_image_delete') - - db.get_image_usage(uuid=uuid).AndReturn(None) - db.get_image_delete(uuid=uuid).AndReturn(delete) - db.create_image_exists( - created_at=utils.str_time_to_unix(str(DUMMY_TIME)), - owner=TENANT_ID_1, - raw=raw, - audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), - audit_period_ending=utils.str_time_to_unix(audit_period_ending), - size=size, - uuid=uuid, - usage=None, - delete=delete, - deleted_at=utils.str_time_to_unix(str(deleted_at))).AndReturn(raw) - - self.mox.ReplayAll() - - notification = GlanceNotification(body, deployment, routing_key, - json_body) - notification.save_exists(raw) - self.mox.VerifyAll() - - def test_save_image_exists_with_usage_not_none(self): - raw = self.mox.CreateMockAnything() - usage = self.mox.CreateMockAnything() - audit_period_beginning = "2013-05-20 17:31:57.939614" - audit_period_ending = "2013-06-20 17:31:57.939614" - size = 123 - uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" - body = { - "event_type": "image.upload", - "timestamp": "2013-06-20 18:31:57.939614", - "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", - "payload": { - "created_at": str(DUMMY_TIME), - "status": "saving", - "audit_period_beginning": audit_period_beginning, - "audit_period_ending": audit_period_ending, - "properties": { - "image_type": "snapshot", - "instance_uuid": INSTANCE_ID_1, - }, - "size": size, - "owner": TENANT_ID_1, - "id": "2df2ccf6-bc1b-4853-aab0-25fda346b3bb", - } - } - deployment = "1" - routing_key = "glance_monitor.info" - json_body = json.dumps([routing_key, body]) - - self.mox.StubOutWithMock(db, 'create_image_exists') - self.mox.StubOutWithMock(db, 'get_image_usage') - self.mox.StubOutWithMock(db, 'get_image_delete') - - db.get_image_usage(uuid=uuid).AndReturn(usage) - db.create_image_exists( - created_at=utils.str_time_to_unix(str(DUMMY_TIME)), - owner=TENANT_ID_1, - raw=raw, - audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), - audit_period_ending=utils.str_time_to_unix(audit_period_ending), - size=size, - uuid=uuid, - usage=usage).AndReturn(raw) - - self.mox.ReplayAll() - - notification = GlanceNotification(body, deployment, routing_key, - json_body) - notification.save_exists(raw) - self.mox.VerifyAll() - def test_save_usage_should_persist_image_usage(self): raw = self.mox.CreateMockAnything() size = 123 @@ -580,3 +424,254 @@ class NotificationTestCase(StacktachBaseTestCase): notification = Notification(body, deployment, routing_key, json_body) self.assertEquals(notification.save(), raw) self.mox.VerifyAll() + + +class GlanceExistsNotificationTestCase(StacktachBaseTestCase): + def setUp(self): + self.mox = mox.Mox() + + def tearDown(self): + self.mox.UnsetStubs() + + def test_save_glancerawdata(self): + raw = self.mox.CreateMockAnything() + audit_period_beginning = "2013-05-20 17:31:57.939614" + audit_period_ending = "2013-06-20 17:31:57.939614" + created_at = "2013-05-20 19:31:57.939614" + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + body = { + "event_type": "image.exists", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "audit_period_beginning": audit_period_beginning, + "audit_period_ending": audit_period_ending, + "owner": TENANT_ID_1, + "images": + [ + { + "created_at": created_at, + "id": uuid, + "size": size, + "status": "saving", + "properties": {"instance_uuid": INSTANCE_ID_1}, + "deleted_at": None, + }, + { + "created_at": str(DUMMY_TIME), + "id": uuid, + "size": size, + "status": "saving", + "properties": {"instance_uuid": INSTANCE_ID_1}, + "deleted_at": None, + } + ] + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json_body = json.dumps([routing_key, body]) + + self.mox.StubOutWithMock(db, 'create_glance_rawdata') + + db.create_glance_rawdata( + deployment="1", + owner="testtenantid1", + json=json_body, + routing_key=routing_key, + when=utils.str_time_to_unix("2013-06-20 18:31:57.939614"), + publisher="glance-api01-r2961.global.preprod-ord.ohthree.com", + event="image.exists", + service="glance-api01-r2961", + host="global.preprod-ord.ohthree.com", + instance=None, + request_id='', + image_type=0, + status=None, + uuid=None).AndReturn(raw) + + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json_body) + notification.save() + self.mox.VerifyAll() + + def test_save_image_exists_with_created_at_but_deleted_at_none(self): + raw = self.mox.CreateMockAnything() + audit_period_beginning = "2013-05-20 17:31:57.939614" + audit_period_ending = "2013-06-20 17:31:57.939614" + created_at = "2013-05-20 19:31:57.939614" + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + body = { + "event_type": "image.exists", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "audit_period_beginning": audit_period_beginning, + "audit_period_ending": audit_period_ending, + "owner": TENANT_ID_1, + "images": + [ + { + "created_at": created_at, + "id": uuid, + "size": size, + "status": "saving", + "properties": {"instance_uuid": INSTANCE_ID_1}, + "deleted_at": None, + }, + { + "created_at": created_at, + "id": uuid, + "size": size, + "status": "saving", + "properties": {"instance_uuid": INSTANCE_ID_1}, + "deleted_at": None, + } + ] + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json_body = json.dumps([routing_key, body]) + + self.mox.StubOutWithMock(db, 'create_image_exists') + self.mox.StubOutWithMock(db, 'get_image_usage') + + for i in range(0, 2): + db.get_image_usage(uuid=uuid).AndReturn(None) + db.create_image_exists( + created_at=utils.str_time_to_unix(created_at), + owner=TENANT_ID_1, + raw=raw, + audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), + audit_period_ending=utils.str_time_to_unix(audit_period_ending), + size=size, + uuid=uuid, + usage=None).AndReturn(raw) + + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json_body) + notification.save_exists(raw) + self.mox.VerifyAll() + + def test_save_image_exists_with_created_at_and_deleted_at(self): + raw = self.mox.CreateMockAnything() + delete = self.mox.CreateMockAnything() + audit_period_beginning = "2013-05-20 17:31:57.939614" + audit_period_ending = "2013-06-20 17:31:57.939614" + created_at = "2013-05-20 19:31:57.939614" + deleted_at = "2013-05-20 21:31:57.939614" + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + body = { + "event_type": "image.exists", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "audit_period_beginning": audit_period_beginning, + "audit_period_ending": audit_period_ending, + "owner": TENANT_ID_1, + "images": + [ + { + "created_at": created_at, + "id": uuid, + "size": size, + "status": "saving", + "properties": {"instance_uuid": INSTANCE_ID_1}, + "deleted_at": deleted_at, + }, + { + "created_at": created_at, + "id": uuid, + "size": size, + "status": "saving", + "properties": {"instance_uuid": INSTANCE_ID_1}, + "deleted_at": deleted_at, + } + ] + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json_body = json.dumps([routing_key, body]) + self.mox.StubOutWithMock(db, 'create_image_exists') + self.mox.StubOutWithMock(db, 'get_image_usage') + self.mox.StubOutWithMock(db, 'get_image_delete') + + for i in range(0, 2): + db.get_image_usage(uuid=uuid).AndReturn(None) + db.get_image_delete(uuid=uuid).AndReturn(delete) + db.create_image_exists( + created_at=utils.str_time_to_unix(created_at), + owner=TENANT_ID_1, + raw=raw, + audit_period_beginning=utils.str_time_to_unix(audit_period_beginning), + audit_period_ending=utils.str_time_to_unix(audit_period_ending), + size=size, + uuid=uuid, + usage=None, + delete=delete, + deleted_at=utils.str_time_to_unix(deleted_at)).AndReturn(raw) + + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json_body) + notification.save_exists(raw) + self.mox.VerifyAll() + + def test_save_image_exists_without_created_at(self): + raw = self.mox.CreateMockAnything() + raw.id = 1 + audit_period_beginning = "2013-05-20 17:31:57.939614" + audit_period_ending = "2013-06-20 17:31:57.939614" + size = 123 + uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb" + body = { + "event_type": "image.exists", + "timestamp": "2013-06-20 18:31:57.939614", + "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "payload": { + "audit_period_beginning": audit_period_beginning, + "audit_period_ending": audit_period_ending, + "owner": TENANT_ID_1, + "images": + [ + { + "created_at": None, + "id": uuid, + "size": size, + "status": "saving", + "properties": {"instance_uuid": INSTANCE_ID_1}, + "deleted_at": None, + }, + { + "created_at": None, + "id": uuid, + "size": size, + "status": "saving", + "properties": {"instance_uuid": INSTANCE_ID_1}, + "deleted_at": None, + } + ] + } + } + deployment = "1" + routing_key = "glance_monitor.info" + json_body = json.dumps([routing_key, body]) + self.mox.StubOutWithMock(stacklog, 'warn') + stacklog.warn("Ignoring exists without created_at. GlanceRawData(1)") + stacklog.warn("Ignoring exists without created_at. GlanceRawData(1)") + self.mox.ReplayAll() + + notification = GlanceNotification(body, deployment, routing_key, + json_body) + notification.save_exists(raw) + self.mox.VerifyAll()