Reset 'Verifying' notifications on verifier start.

If the stacktach verifier crashes, notifications
'in-flight' can be stuck in 'verifying' status'.

This change flips those back to 'pending' so they get
processed.

Change-Id: Ie4aabed0c4991429a3e18e3b28813917d822867a

verifier/base_verifier.py

@@ -166,6 +166,7 @@ class Verifier(object):
 
     def run(self):
         logger = _get_child_logger()
+        self.run_startup()
         if self.enable_notifications:
             exchange_name = self.exchange()
             exchange = message_service.create_exchange(
@@ -219,6 +220,9 @@ class Verifier(object):
             self._run()
         return False
 
+    def run_startup(self):
+        pass
+
     def verify_for_range(self, ending_max, callback=None):
         pass
 

verifier/nova_verifier.py

@@ -356,6 +356,16 @@ class NovaVerifier(base_verifier.Verifier):
                                    models.InstanceExists.VERIFYING)
         return count+sent_unverified_count
 
+    def run_startup(self):
+        logger = _get_child_logger()
+        exists = models.InstanceExists.find(
+            ending_max=self._utcnow(), status=models.InstanceExists.VERIFYING)
+        count = exists.count()
+        if count > 0:
+            msg = "nova: Cleaning up %s exists stuck in VERIFYING status..." % count
+            logger.info(msg)
+            exists.update(status=models.InstanceExists.PENDING)
+
     def reconcile_failed(self):
         for failed_exist in self.failed:
             self.reconciler.failed_validation(failed_exist)