x/stacktach
Code Issues Proposed changes

Adding fail_reason to exists table, tweaking verifier

Browse Source
This commit is contained in:
Andrew Melton 2013-03-21 14:12:20 -04:00
parent 5b40312752
commit 0def04b4ca
4 changed files with 53 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
migrations/006_delta.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE stacktach_instanceexists ADD fail_reason VARCHAR(200);
CREATE INDEX `stacktach_instanceexists_fail_reason1` ON `stacktach_instanceexists` (`fail_reason`);

2
stacktach/models.py
View File

@ -124,6 +124,8 @@ class InstanceExists(models.Model):
status = models.CharField(max_length=50, db_index=True, status = models.CharField(max_length=50, db_index=True,
choices=STATUS_CHOICES, choices=STATUS_CHOICES,
default=PENDING) default=PENDING)
fail_reason = models.CharField(max_length=2000, null=True,
blank=True, db_index=True)
raw = models.ForeignKey(RawData, related_name='+', null=True) raw = models.ForeignKey(RawData, related_name='+', null=True)
usage = models.ForeignKey(InstanceUsage, related_name='+', null=True) usage = models.ForeignKey(InstanceUsage, related_name='+', null=True)
delete = models.ForeignKey(InstanceDeletes, related_name='+', null=True) delete = models.ForeignKey(InstanceDeletes, related_name='+', null=True)

31
tests/unit/test_verifier_db.py
View File

@ -319,12 +319,12 @@ class VerifierTestCase(unittest.TestCase):
self.assertEqual(fm.actual, decimal.Decimal('6.1')) self.assertEqual(fm.actual, decimal.Decimal('6.1'))
self.mox.VerifyAll() self.mox.VerifyAll()
def test_verify(self): def test_verify_pass(self):
exist = self.mox.CreateMockAnything() exist = self.mox.CreateMockAnything()
exist.launched_at = decimal.Decimal('1.1') exist.launched_at = decimal.Decimal('1.1')
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
dbverifier._verify_for_launch(exist) dbverifier._verify_for_launch(exist)
dbverifier._verify_for_delete(exist) dbverifier._verify_for_delete(exist)
@ -333,13 +333,15 @@ class VerifierTestCase(unittest.TestCase):
dbverifier._verify(exist) dbverifier._verify(exist)
self.mox.VerifyAll() self.mox.VerifyAll()
def test_verify(self): def test_verify_no_launched_at(self):
exist = self.mox.CreateMockAnything() exist = self.mox.CreateMockAnything()
exist.launched_at = None
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
dbverifier._mark_exists_failed(exist) dbverifier._mark_exist_failed(exist,
reason="Exists without a launched_at")
self.mox.ReplayAll() self.mox.ReplayAll()
dbverifier._verify(exist) dbverifier._verify(exist)
self.mox.VerifyAll() self.mox.VerifyAll()
@ -349,11 +351,11 @@ class VerifierTestCase(unittest.TestCase):
exist.launched_at = decimal.Decimal('1.1') exist.launched_at = decimal.Decimal('1.1')
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
verify_exception = VerificationException('test') verify_exception = VerificationException('test')
dbverifier._verify_for_launch(exist).AndRaise(verify_exception) dbverifier._verify_for_launch(exist).AndRaise(verify_exception)
dbverifier._mark_exists_failed(exist) dbverifier._mark_exist_failed(exist, reason='test')
self.mox.ReplayAll() self.mox.ReplayAll()
dbverifier._verify(exist) dbverifier._verify(exist)
self.mox.VerifyAll() self.mox.VerifyAll()
@ -363,12 +365,12 @@ class VerifierTestCase(unittest.TestCase):
exist.launched_at = decimal.Decimal('1.1') exist.launched_at = decimal.Decimal('1.1')
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
verify_exception = VerificationException('test') verify_exception = VerificationException('test')
dbverifier._verify_for_launch(exist) dbverifier._verify_for_launch(exist)
dbverifier._verify_for_delete(exist).AndRaise(verify_exception) dbverifier._verify_for_delete(exist).AndRaise(verify_exception)
dbverifier._mark_exists_failed(exist) dbverifier._mark_exist_failed(exist, reason='test')
self.mox.ReplayAll() self.mox.ReplayAll()
dbverifier._verify(exist) dbverifier._verify(exist)
self.mox.VerifyAll() self.mox.VerifyAll()
@ -378,11 +380,10 @@ class VerifierTestCase(unittest.TestCase):
exist.launched_at = decimal.Decimal('1.1') exist.launched_at = decimal.Decimal('1.1')
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
dbverifier._verify_for_launch(exist).AndRaise(Exception()) dbverifier._verify_for_launch(exist).AndRaise(Exception())
dbverifier._mark_exists_failed(exist) dbverifier._mark_exist_failed(exist, reason='Exception')
self.mox.ReplayAll() self.mox.ReplayAll()
dbverifier._verify(exist) dbverifier._verify(exist)
self.mox.VerifyAll() self.mox.VerifyAll()
@ -392,11 +393,11 @@ class VerifierTestCase(unittest.TestCase):
exist.launched_at = decimal.Decimal('1.1') exist.launched_at = decimal.Decimal('1.1')
self.mox.StubOutWithMock(dbverifier, '_verify_for_launch') self.mox.StubOutWithMock(dbverifier, '_verify_for_launch')
self.mox.StubOutWithMock(dbverifier, '_verify_for_delete') self.mox.StubOutWithMock(dbverifier, '_verify_for_delete')
self.mox.StubOutWithMock(dbverifier, '_mark_exists_failed') self.mox.StubOutWithMock(dbverifier, '_mark_exist_failed')
self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified') self.mox.StubOutWithMock(dbverifier, '_mark_exist_verified')
dbverifier._verify_for_launch(exist) dbverifier._verify_for_launch(exist)
dbverifier._verify_for_delete(exist).AndRaise(Exception()) dbverifier._verify_for_delete(exist).AndRaise(Exception())
dbverifier._mark_exists_failed(exist) dbverifier._mark_exist_failed(exist, reason='Exception')
self.mox.ReplayAll() self.mox.ReplayAll()
dbverifier._verify(exist) dbverifier._verify(exist)
self.mox.VerifyAll() self.mox.VerifyAll()
@ -417,6 +418,7 @@ class VerifierTestCase(unittest.TestCase):
results.count().AndReturn(2) results.count().AndReturn(2)
exist1 = self.mox.CreateMockAnything() exist1 = self.mox.CreateMockAnything()
exist2 = self.mox.CreateMockAnything() exist2 = self.mox.CreateMockAnything()
results.__getslice__(0, 1000).AndReturn(results)
results.__iter__().AndReturn([exist1, exist2].__iter__()) results.__iter__().AndReturn([exist1, exist2].__iter__())
exist1.save() exist1.save()
exist2.save() exist2.save()
@ -445,6 +447,7 @@ class VerifierTestCase(unittest.TestCase):
results.count().AndReturn(2) results.count().AndReturn(2)
exist1 = self.mox.CreateMockAnything() exist1 = self.mox.CreateMockAnything()
exist2 = self.mox.CreateMockAnything() exist2 = self.mox.CreateMockAnything()
results.__getslice__(0, 1000).AndReturn(results)
results.__iter__().AndReturn([exist1, exist2].__iter__()) results.__iter__().AndReturn([exist1, exist2].__iter__())
exist1.save() exist1.save()
exist2.save() exist2.save()

49
verifier/dbverifier.py
View File

@ -72,8 +72,10 @@ def _mark_exist_verified(exist):
exist.save() exist.save()
def _mark_exists_failed(exist): def _mark_exist_failed(exist, reason=None):
exist.status = models.InstanceExists.FAILED exist.status = models.InstanceExists.FAILED
if reason:
exist.fail_reason = reason
exist.save() exist.save()
@ -114,12 +116,15 @@ def _verify_for_launch(exist):
.filter(instance=exist.instance).count() > 0: .filter(instance=exist.instance).count() > 0:
launches = _find_launch(exist.instance, launches = _find_launch(exist.instance,
dt.dt_from_decimal(exist.launched_at)) dt.dt_from_decimal(exist.launched_at))
if launches.count() != 1: count = launches.count()
query = { query = {
'instance': exist.instance, 'instance': exist.instance,
'launched_at': exist.launched_at 'launched_at': exist.launched_at
} }
if count > 1:
raise AmbiguousResults('InstanceUsage', query) raise AmbiguousResults('InstanceUsage', query)
elif count == 0:
raise NotFound('InstanceUsage', query)
launch = launches[0] launch = launches[0]
else: else:
raise NotFound('InstanceUsage', {'instance': exist.instance}) raise NotFound('InstanceUsage', {'instance': exist.instance})
@ -190,10 +195,10 @@ def _verify(exist):
verified = True verified = True
_mark_exist_verified(exist) _mark_exist_verified(exist)
except VerificationException: except VerificationException, e:
_mark_exists_failed(exist) _mark_exist_failed(exist, reason=str(e))
except Exception, e: except Exception, e:
_mark_exists_failed(exist) _mark_exist_failed(exist, reason=e.__class__.__name__)
LOG.exception(e) LOG.exception(e)
return verified, exist return verified, exist
@ -206,11 +211,21 @@ def verify_for_range(pool, when_max, callback=None):
exists = _list_exists(received_max=when_max, exists = _list_exists(received_max=when_max,
status=models.InstanceExists.PENDING) status=models.InstanceExists.PENDING)
count = exists.count() count = exists.count()
for exist in exists: added = 0
exist.status = models.InstanceExists.VERIFYING update_interval = datetime.timedelta(seconds=30)
exist.save() next_update = datetime.datetime.utcnow() + update_interval
result = pool.apply_async(_verify, args=(exist,), callback=callback) LOG.info("Adding %s exists to queue." % count)
results.append(result) while added < count:
for exist in exists[0:1000]:
exist.status = models.InstanceExists.VERIFYING
exist.save()
result = pool.apply_async(_verify, args=(exist,),
callback=callback)
results.append(result)
added += 1
if datetime.datetime.utcnow() > next_update:
LOG.info("Added %s exists to queue." % added)
next_update = datetime.datetime.utcnow() + update_interval
return count return count
@ -236,9 +251,9 @@ def clean_results():
def _send_notification(message, routing_key, connection, exchange): def _send_notification(message, routing_key, connection, exchange):
with kombu.pools.producers[connection].acquire(block=True) as producer: with kombu.pools.producers[connection].acquire(block=True) as producer:
kombu.common.maybe_declare(exchange, producer.channel) kombu.common.maybe_declare(exchange, producer.channel)
producer.publish(message, routing_key) producer.publish(message, routing_key)
def send_verified_notification(exist, connection, exchange): def send_verified_notification(exist, connection, exchange):