Daemon PID file fixes

Daemons hang if a PID file exists

* If there is a stale PID file (after kill -9 for example) then cleanup
* If there is a current running daemon then die (and log if possible)

Fixes bug #1153737

Change-Id: I1ccdfeba70e866922d4214466811ef596d1ba892

libra/mgm/mgm.py

@@ -14,6 +14,7 @@
 
 import daemon
 import daemon.pidfile
+import daemon.runner
 import grp
 import pwd
 import signal
@@ -21,6 +22,7 @@ import time
 import sys
 import os
 import threading
+import lockfile
 
 from novaclient import exceptions
 from libra.openstack.common import importutils
@@ -424,10 +426,14 @@ def main():
     if args.nodaemon:
         server.main()
     else:
+        pidfile = daemon.pidfile.TimeoutPIDLockFile(args.pid, 10)
+        if daemon.runner.is_pidfile_stale(pidfile):
+            logger.warning("Cleaning up stale PID file")
+            pidfile.break_lock()
         context = daemon.DaemonContext(
             working_directory='/',
             umask=0o022,
-            pidfile=daemon.pidfile.TimeoutPIDLockFile(args.pid),
+            pidfile=pidfile,
             files_preserve=[logger.handlers[0].stream]
         )
         if args.user:
@@ -445,7 +451,15 @@ def main():
             except KeyError:
                 logger.critical("Invalid group: %s" % args.group)
                 return 1
-        with context:
-            server.main()
+        try:
+            context.open()
+        except lockfile.LockTimeout:
+            logger.critical(
+                "Failed to lock pidfile %s, another instance running?",
+                args.pid
+            )
+            return 1
+
+        server.main()
 
     return 0

libra/statsd/main.py

@@ -14,6 +14,8 @@
 
 import daemon
 import daemon.pidfile
+import daemon.runner
+import lockfile
 import gearman.errors
 import grp
 import json
@@ -96,9 +98,13 @@ def main():
     if args.nodaemon:
         start(logger, args.server)
     else:
+        pidfile = daemon.pidfile.TimeoutPIDLockFile(args.pid, 10)
+        if daemon.runner.is_pidfile_stale(pidfile):
+            logger.warning("Cleaning up stale PID file")
+            pidfile.break_lock()
         context = daemon.DaemonContext(
             umask=0o022,
-            pidfile=daemon.pidfile.TimeoutPIDLockFile(args.pid),
+            pidfile=pidfile,
             files_preserve=[logger.handlers[0].stream]
         )
         if args.user:
@@ -117,5 +123,12 @@ def main():
                 logger.critical("Invalid group: %s" % args.group)
                 return 1
 
-        with context:
-            start(logger, args.server)
+        try:
+            context.open()
+        except lockfile.LockTimeout:
+            logger.critical(
+                "Failed to lock pidfile %s, another instance running?",
+                args.pid
+            )
+
+        start(logger, args.server)

libra/worker/main.py

@@ -16,8 +16,10 @@ import eventlet
 eventlet.monkey_patch()
 
 import daemon
+import lockfile
 import os
 import daemon.pidfile
+import daemon.runner
 import grp
 import pwd
 
@@ -126,10 +128,14 @@ def main():
     if args.nodaemon:
         server.main(task_list)
     else:
+        pidfile = daemon.pidfile.TimeoutPIDLockFile(args.pid, 10)
+        if daemon.runner.is_pidfile_stale(pidfile):
+            logger.warning("Cleaning up stale PID file")
+            pidfile.break_lock()
         context = daemon.DaemonContext(
             working_directory='/etc/haproxy',
             umask=0o022,
-            pidfile=daemon.pidfile.TimeoutPIDLockFile(args.pid),
+            pidfile=pidfile,
             files_preserve=[logger.handlers[0].stream]
         )
         if args.user:
@@ -148,7 +154,15 @@ def main():
                 logger.critical("Invalid group: %s" % args.group)
                 return 1
 
-        with context:
-            server.main(task_list)
+        try:
+            context.open()
+        except lockfile.LockTimeout:
+            logger.critical(
+                "Failed to lock pidfile %s, another instance running?",
+                args.pid
+            )
+            return 1
+
+        server.main(task_list)
 
     return 0