From 0def04b4ca712dfac9f82376b817c67539ec5c66 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Thu, 21 Mar 2013 14:12:20 -0400 Subject: [PATCH] Adding fail_reason to exists table, tweaking verifier --- migrations/006_delta.sql | 2 ++ stacktach/models.py | 2 ++ tests/unit/test_verifier_db.py | 31 +++++++++++---------- verifier/dbverifier.py | 49 ++++++++++++++++++++++------------ 4 files changed, 53 insertions(+), 31 deletions(-) create mode 100644 migrations/006_delta.sql diff --git a/migrations/006_delta.sql b/migrations/006_delta.sql new file mode 100644 index 0000000..60d7863 --- /dev/null +++ b/migrations/006_delta.sql @@ -0,0 +1,2 @@ +ALTER TABLE stacktach_instanceexists ADD fail_reason VARCHAR(200); +CREATE INDEX `stacktach_instanceexists_fail_reason1` ON `stacktach_instanceexists` (`fail_reason`); \ No newline at end of file diff --git a/stacktach/models.py b/stacktach/models.py index 935f019..fd284bb 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -124,6 +124,8 @@ class InstanceExists(models.Model): status = models.CharField(max_length=50, db_index=True, choices=STATUS_CHOICES, default=PENDING) + fail_reason = models.CharField(max_length=2000, null=True, + blank=True, db_index=True) raw = models.ForeignKey(RawData, related_name='+', null=True) usage = models.ForeignKey(InstanceUsage, related_name='+', null=True) delete = models.ForeignKey(InstanceDeletes, related_name='+', null=True) diff --git a/tests/unit/test_verifier_db.py b/tests/unit/test_verifier_db.py index 529b4ad..b96fb14 100644 --- a/tests/unit/test_verifier_db.py +++ b/tests/unit/test_verifier_db.py @@ -319,12 +319,12 @@ class VerifierTestCase(unittest.TestCase): self.assertEqual(fm.actual, decimal.Decimal('6.1')) self.mox.VerifyAll() - def test_verify(self): + def test_verify_pass(self): exist = self.mox.CreateMockAnything() exist.launched_at = decimal.Decimal('1.1') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') - self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') dbverifier._verify_for_launch(exist) dbverifier._verify_for_delete(exist) @@ -333,13 +333,15 @@ class VerifierTestCase(unittest.TestCase): dbverifier._verify(exist) self.mox.VerifyAll() - def test_verify(self): + def test_verify_no_launched_at(self): exist = self.mox.CreateMockAnything() + exist.launched_at = None self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') - self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') - dbverifier._mark_exists_failed(exist) + dbverifier._mark_exist_failed(exist, + reason="Exists without a launched_at") self.mox.ReplayAll() dbverifier._verify(exist) self.mox.VerifyAll() @@ -349,11 +351,11 @@ class VerifierTestCase(unittest.TestCase): exist.launched_at = decimal.Decimal('1.1') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') - self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') verify_exception = VerificationException('test') dbverifier._verify_for_launch(exist).AndRaise(verify_exception) - dbverifier._mark_exists_failed(exist) + dbverifier._mark_exist_failed(exist, reason='test') self.mox.ReplayAll() dbverifier._verify(exist) self.mox.VerifyAll() @@ -363,12 +365,12 @@ class VerifierTestCase(unittest.TestCase): exist.launched_at = decimal.Decimal('1.1') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') - self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') verify_exception = VerificationException('test') dbverifier._verify_for_launch(exist) dbverifier._verify_for_delete(exist).AndRaise(verify_exception) - dbverifier._mark_exists_failed(exist) + dbverifier._mark_exist_failed(exist, reason='test') self.mox.ReplayAll() dbverifier._verify(exist) self.mox.VerifyAll() @@ -378,11 +380,10 @@ class VerifierTestCase(unittest.TestCase): exist.launched_at = decimal.Decimal('1.1') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') - self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') - dbverifier._verify_for_launch(exist).AndRaise(Exception()) - dbverifier._mark_exists_failed(exist) + dbverifier._mark_exist_failed(exist, reason='Exception') self.mox.ReplayAll() dbverifier._verify(exist) self.mox.VerifyAll() @@ -392,11 +393,11 @@ class VerifierTestCase(unittest.TestCase): exist.launched_at = decimal.Decimal('1.1') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') - self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') + self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') dbverifier._verify_for_launch(exist) dbverifier._verify_for_delete(exist).AndRaise(Exception()) - dbverifier._mark_exists_failed(exist) + dbverifier._mark_exist_failed(exist, reason='Exception') self.mox.ReplayAll() dbverifier._verify(exist) self.mox.VerifyAll() @@ -417,6 +418,7 @@ class VerifierTestCase(unittest.TestCase): results.count().AndReturn(2) exist1 = self.mox.CreateMockAnything() exist2 = self.mox.CreateMockAnything() + results.__getslice__(0, 1000).AndReturn(results) results.__iter__().AndReturn([exist1, exist2].__iter__()) exist1.save() exist2.save() @@ -445,6 +447,7 @@ class VerifierTestCase(unittest.TestCase): results.count().AndReturn(2) exist1 = self.mox.CreateMockAnything() exist2 = self.mox.CreateMockAnything() + results.__getslice__(0, 1000).AndReturn(results) results.__iter__().AndReturn([exist1, exist2].__iter__()) exist1.save() exist2.save() diff --git a/verifier/dbverifier.py b/verifier/dbverifier.py index 1c84f83..e98afb9 100644 --- a/verifier/dbverifier.py +++ b/verifier/dbverifier.py @@ -72,8 +72,10 @@ def _mark_exist_verified(exist): exist.save() -def _mark_exists_failed(exist): +def _mark_exist_failed(exist, reason=None): exist.status = models.InstanceExists.FAILED + if reason: + exist.fail_reason = reason exist.save() @@ -114,12 +116,15 @@ def _verify_for_launch(exist): .filter(instance=exist.instance).count() > 0: launches = _find_launch(exist.instance, dt.dt_from_decimal(exist.launched_at)) - if launches.count() != 1: - query = { - 'instance': exist.instance, - 'launched_at': exist.launched_at - } + count = launches.count() + query = { + 'instance': exist.instance, + 'launched_at': exist.launched_at + } + if count > 1: raise AmbiguousResults('InstanceUsage', query) + elif count == 0: + raise NotFound('InstanceUsage', query) launch = launches[0] else: raise NotFound('InstanceUsage', {'instance': exist.instance}) @@ -190,10 +195,10 @@ def _verify(exist): verified = True _mark_exist_verified(exist) - except VerificationException: - _mark_exists_failed(exist) + except VerificationException, e: + _mark_exist_failed(exist, reason=str(e)) except Exception, e: - _mark_exists_failed(exist) + _mark_exist_failed(exist, reason=e.__class__.__name__) LOG.exception(e) return verified, exist @@ -206,11 +211,21 @@ def verify_for_range(pool, when_max, callback=None): exists = _list_exists(received_max=when_max, status=models.InstanceExists.PENDING) count = exists.count() - for exist in exists: - exist.status = models.InstanceExists.VERIFYING - exist.save() - result = pool.apply_async(_verify, args=(exist,), callback=callback) - results.append(result) + added = 0 + update_interval = datetime.timedelta(seconds=30) + next_update = datetime.datetime.utcnow() + update_interval + LOG.info("Adding %s exists to queue." % count) + while added < count: + for exist in exists[0:1000]: + exist.status = models.InstanceExists.VERIFYING + exist.save() + result = pool.apply_async(_verify, args=(exist,), + callback=callback) + results.append(result) + added += 1 + if datetime.datetime.utcnow() > next_update: + LOG.info("Added %s exists to queue." % added) + next_update = datetime.datetime.utcnow() + update_interval return count @@ -236,9 +251,9 @@ def clean_results(): def _send_notification(message, routing_key, connection, exchange): - with kombu.pools.producers[connection].acquire(block=True) as producer: - kombu.common.maybe_declare(exchange, producer.channel) - producer.publish(message, routing_key) + with kombu.pools.producers[connection].acquire(block=True) as producer: + kombu.common.maybe_declare(exchange, producer.channel) + producer.publish(message, routing_key) def send_verified_notification(exist, connection, exchange):