From 9f5468df7a1a37d16112feb2cd95bc39affc09e5 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 20 May 2013 12:59:22 -0400 Subject: [PATCH] Starting reconcile in verifier --- stacktach/models.py | 2 + tests/unit/test_verifier_db.py | 157 +++++++++++++++++++++++++++++++-- verifier/dbverifier.py | 111 +++++++++++++++++++---- 3 files changed, 248 insertions(+), 22 deletions(-) diff --git a/stacktach/models.py b/stacktach/models.py index 6236695..00d233c 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -120,11 +120,13 @@ class InstanceExists(models.Model): PENDING = 'pending' VERIFYING = 'verifying' VERIFIED = 'verified' + RECONCILED = 'reconciled' FAILED = 'failed' STATUS_CHOICES = [ (PENDING, 'Pending Verification'), (VERIFYING, 'Currently Being Verified'), (VERIFIED, 'Passed Verification'), + (RECONCILED, 'Passed Verification After Reconciliation'), (FAILED, 'Failed Verification'), ] instance = models.CharField(max_length=50, null=True, diff --git a/tests/unit/test_verifier_db.py b/tests/unit/test_verifier_db.py index 54f7def..a10e424 100644 --- a/tests/unit/test_verifier_db.py +++ b/tests/unit/test_verifier_db.py @@ -64,6 +64,9 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(models, 'InstanceDeletes', use_mock_anything=True) models.InstanceDeletes.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceReconcile', + use_mock_anything=True) + models.InstanceReconcile.objects = self.mox.CreateMockAnything() self.mox.StubOutWithMock(models, 'InstanceExists', use_mock_anything=True) models.InstanceExists.objects = self.mox.CreateMockAnything() @@ -369,7 +372,8 @@ class VerifierTestCase(unittest.TestCase): dbverifier._verify_for_delete(exist) dbverifier._mark_exist_verified(exist) self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertTrue(result) self.mox.VerifyAll() def test_verify_no_launched_at(self): @@ -381,8 +385,12 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') dbverifier._mark_exist_failed(exist, reason="Exists without a launched_at") + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, mox.IgnoreArg())\ + .AndRaise(NotFound('InstanceReconcile', {})) self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_launch_fail(self): @@ -394,9 +402,140 @@ class VerifierTestCase(unittest.TestCase): self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') verify_exception = VerificationException('test') dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(NotFound('InstanceReconcile', {})) dbverifier._mark_exist_failed(exist, reason='test') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_verify_fail_reconcile_success(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception) + dbverifier._mark_exist_verified(exist) + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertTrue(result) + self.mox.VerifyAll() + + def test_verify_fail_no_reconciled_data_successful_reconcile(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + rec_exception = NotFound("InstanceReconcile", {}) + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(rec_exception) + self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') + dbverifier._attempt_reconciliation(exist, rec_exception)\ + .AndReturn(True) + dbverifier._mark_exist_verified(exist, + reconciled=True, + reason='test') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertTrue(result) + self.mox.VerifyAll() + + def test_verify_fail_no_reconciled_data_unsuccessful_reconcile(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + rec_exception = NotFound("InstanceReconcile", {}) + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(rec_exception) + self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') + dbverifier._attempt_reconciliation(exist, rec_exception)\ + .AndReturn(False) + dbverifier._mark_exist_failed(exist, reason='test') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_verify_fail_bad_reconciled_data_successful_reconcile(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + rec_exception = VerificationException("test2") + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(rec_exception) + self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') + dbverifier._attempt_reconciliation(exist, rec_exception)\ + .AndReturn(True) + dbverifier._mark_exist_verified(exist, + reconciled=True, + reason='test2') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertTrue(result) + self.mox.VerifyAll() + + def test_verify_fail_bad_reconciled_data_unsuccessful_reconcile(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + rec_exception = VerificationException("test2") + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(rec_exception) + self.mox.StubOutWithMock(dbverifier, '_attempt_reconciliation') + dbverifier._attempt_reconciliation(exist, rec_exception)\ + .AndReturn(False) + dbverifier._mark_exist_failed(exist, reason='test2') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_verify_fail_with_reconciled_data_exception(self): + exist = self.mox.CreateMockAnything() + exist.launched_at = decimal.Decimal('1.1') + self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') + self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') + verify_exception = VerificationException('test') + dbverifier._verify_for_launch(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(Exception()) + dbverifier._mark_exist_failed(exist, reason='Exception') + self.mox.ReplayAll() + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_delete_fail(self): @@ -409,9 +548,13 @@ class VerifierTestCase(unittest.TestCase): verify_exception = VerificationException('test') dbverifier._verify_for_launch(exist) dbverifier._verify_for_delete(exist).AndRaise(verify_exception) + self.mox.StubOutWithMock(dbverifier, '_verify_with_reconciled_data') + dbverifier._verify_with_reconciled_data(exist, verify_exception)\ + .AndRaise(NotFound('InstanceReconcile', {})) dbverifier._mark_exist_failed(exist, reason='test') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_exception_during_launch(self): @@ -424,7 +567,8 @@ class VerifierTestCase(unittest.TestCase): dbverifier._verify_for_launch(exist).AndRaise(Exception()) dbverifier._mark_exist_failed(exist, reason='Exception') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_exception_during_delete(self): @@ -438,7 +582,8 @@ class VerifierTestCase(unittest.TestCase): dbverifier._verify_for_delete(exist).AndRaise(Exception()) dbverifier._mark_exist_failed(exist, reason='Exception') self.mox.ReplayAll() - dbverifier._verify(exist) + result, exists = dbverifier._verify(exist) + self.assertFalse(result) self.mox.VerifyAll() def test_verify_for_range_without_callback(self): diff --git a/verifier/dbverifier.py b/verifier/dbverifier.py index 7a6f1f8..858877b 100644 --- a/verifier/dbverifier.py +++ b/verifier/dbverifier.py @@ -69,6 +69,15 @@ def _find_launch(instance, launched): return models.InstanceUsage.objects.filter(**params) +def _find_reconcile(instance, launched): + start = launched - datetime.timedelta(microseconds=launched.microsecond) + end = start + datetime.timedelta(microseconds=999999) + params = {'instance': instance, + 'launched_at__gte': dt.dt_to_decimal(start), + 'launched_at__lte': dt.dt_to_decimal(end)} + return models.InstanceReconcile.objects.filter(**params) + + def _find_delete(instance, launched, deleted_max=None): start = launched - datetime.timedelta(microseconds=launched.microsecond) end = start + datetime.timedelta(microseconds=999999) @@ -80,8 +89,16 @@ def _find_delete(instance, launched, deleted_max=None): return models.InstanceDeletes.objects.filter(**params) -def _mark_exist_verified(exist): - exist.status = models.InstanceExists.VERIFIED +def _mark_exist_verified(exist, + reconciled=False, + reason=None): + if not reconciled: + exist.status = models.InstanceExists.VERIFIED + else: + exist.status = models.InstanceExists.RECONCILED + if reason is not None: + exist.fail_reason = reason + exist.save() @@ -136,10 +153,11 @@ def _verify_field_mismatch(exists, launch): launch.tenant) -def _verify_for_launch(exist): - if exist.usage: +def _verify_for_launch(exist, launch=None, launch_type="InstanceUsage"): + + if not launch and exist.usage: launch = exist.usage - else: + elif not launch: if models.InstanceUsage.objects\ .filter(instance=exist.instance).count() > 0: launches = _find_launch(exist.instance, @@ -150,23 +168,22 @@ def _verify_for_launch(exist): 'launched_at': exist.launched_at } if count > 1: - raise AmbiguousResults('InstanceUsage', query) + raise AmbiguousResults(launch_type, query) elif count == 0: - raise NotFound('InstanceUsage', query) + raise NotFound(launch_type, query) launch = launches[0] else: - raise NotFound('InstanceUsage', {'instance': exist.instance}) + raise NotFound(launch_type, {'instance': exist.instance}) _verify_field_mismatch(exist, launch) -def _verify_for_delete(exist): +def _verify_for_delete(exist, delete=None, delete_type="InstanceDelete"): - delete = None - if exist.delete: + if not delete and exist.delete: # We know we have a delete and we have it's id delete = exist.delete - else: + elif not delete: if exist.deleted_at: # We received this exists before the delete, go find it deletes = _find_delete(exist.instance, @@ -178,7 +195,7 @@ def _verify_for_delete(exist): 'instance': exist.instance, 'launched_at': exist.launched_at } - raise NotFound('InstanceDelete', query) + raise NotFound(delete_type, query) else: # We don't know if this is supposed to have a delete or not. # Thus, we need to check if we have a delete for this instance. @@ -190,7 +207,7 @@ def _verify_for_delete(exist): deleted_at_max = dt.dt_from_decimal(exist.audit_period_ending) deletes = _find_delete(exist.instance, launched_at, deleted_at_max) if deletes.count() > 0: - reason = 'Found InstanceDeletes for non-delete exist' + reason = 'Found %ss for non-delete exist' % delete_type raise VerificationException(reason) if delete: @@ -205,6 +222,35 @@ def _verify_for_delete(exist): delete.deleted_at) +def _verify_with_reconciled_data(exist, ex): + if not exist.launched_at: + raise VerificationException("Exists without a launched_at") + + query = models.InstanceReconcile.objects.filter(instance=exist.instance) + if query.count() > 0: + recs = _find_reconcile(exist.instance, + dt.dt_from_decimal(exist.launched_at)) + search_query = {'instance': exist.instance, + 'launched_at': exist.launched_at} + count = recs.count() + if count > 1: + raise AmbiguousResults('InstanceReconcile', search_query) + elif count == 0: + raise NotFound('InstanceReconcile', search_query) + reconcile = recs[0] + else: + raise NotFound('InstanceReconcile', {'instance': exist.instance}) + + _verify_for_launch(exist, launch=reconcile, + launch_type="InstanceReconcile") + _verify_for_delete(exist, delete=reconcile, + delete_type="InstanceReconcile") + + +def _attempt_reconciliation(exists, ex): + pass + + def _verify(exist): verified = False try: @@ -216,8 +262,41 @@ def _verify(exist): verified = True _mark_exist_verified(exist) - except VerificationException, e: - _mark_exist_failed(exist, reason=str(e)) + except VerificationException, orig_e: + # Something is wrong with the InstanceUsage record + try: + # Attempt to verify against reconciled data + _verify_with_reconciled_data(exist, orig_e) + verified = True + _mark_exist_verified(exist) + except NotFound, rec_e: + # No reconciled data available, so let's try to reconcile + if _attempt_reconciliation(exist, rec_e): + # We were able to reconcile the data, but we still need + # to record why it originally failed + verified = True + _mark_exist_verified(exist, + reconciled=True, + reason=str(orig_e)) + else: + # Couldn't reconcile the data, just mark it failed + _mark_exist_failed(exist, reason=str(orig_e)) + except VerificationException, rec_e: + # Reconciled data was available, but it's wrong as well + # Let's try to reconcile again + if _attempt_reconciliation(exist, rec_e): + # We were able to reconcile the data, but we still need + # to record why it failed again + verified = True + _mark_exist_verified(exist, + reconciled=True, + reason=str(rec_e)) + else: + # Couldn't reconcile the data, just mark it failed + _mark_exist_failed(exist, reason=str(rec_e)) + except Exception, rec_e: + _mark_exist_failed(exist, reason=rec_e.__class__.__name__) + LOG.exception(rec_e) except Exception, e: _mark_exist_failed(exist, reason=e.__class__.__name__) LOG.exception(e)