Add syslog options and worker support.

Adds options for logging to syslog. As a consequence of this, we
can no longer log BEFORE we become a daemon. It doesn't work correctly
otherwise. Worker, statsd and pool manager support this new behavior
with this change.

Note that instead of logging exceptions that occur before becoming a
daemon, we simply let them propogate out.

Change-Id: If27da7fc1f7a51dba84bb5ecb074a657c3785164

doc/config.rst

@@ -137,6 +137,18 @@ Worker Command Line Options
       Used to specify the Gearman job server hostname and port. This option
       can be used multiple times to specify multiple job servers.
 
+   .. option:: --syslog
+
+      Send log events to syslog.
+
+   .. option:: --syslog-socket
+
+      Socket to use for the syslog connection. Default is */dev/log*.
+
+   .. option:: --syslog-facility
+
+      Syslog logging facility. Default is *LOCAL7*.
+
    .. option:: --user <USER>
 
       Specifies the user for the process when in daemon mode. Default is the
@@ -263,6 +275,18 @@ Pool Manager Command Line Options
       Name of the PID file to use. Default is:
       */var/run/libra/libra_worker.pid*
 
+   .. option:: --syslog
+
+      Send log events to syslog.
+
+   .. option:: --syslog-socket
+
+      Socket to use for the syslog connection. Default is */dev/log*.
+
+   .. option:: --syslog-facility
+
+      Syslog logging facility. Default is *LOCAL7*.
+
    .. option:: --user <USER>
 
       Specifies the user for the process when in daemon mode. Default is the
@@ -320,3 +344,16 @@ Statsd Command Line Options
    .. option:: --datadog_tags <TAGS>
 
       A list of tags to be used for the datadog driver
+
+   .. option:: --syslog
+
+      Send log events to syslog.
+
+   .. option:: --syslog-socket
+
+      Socket to use for the syslog connection. Default is */dev/log*.
+
+   .. option:: --syslog-facility
+
+      Syslog logging facility. Default is *LOCAL7*.
+

libra/common/options.py

@@ -14,7 +14,9 @@
 import argparse
 import logging
 import logging.handlers
+import os
 import os.path
+import pwd
 import sys
 import ConfigParser
 
@@ -122,6 +124,20 @@ class Options(object):
             '-n', '--nodaemon', dest='nodaemon', action='store_true',
             help='do not run in daemon mode'
         )
+        self.parser.add_argument(
+            '--syslog', dest='syslog', action='store_true',
+            help='use syslog for logging output'
+        )
+        self.parser.add_argument(
+            '--syslog-socket', dest='syslog_socket',
+            default='/dev/log',
+            help='socket to use for syslog connection (default: /dev/log)'
+        )
+        self.parser.add_argument(
+            '--syslog-facility', dest='syslog_facility',
+            default='local7',
+            help='syslog logging facility (default: local7)'
+        )
         self.parser.add_argument(
             '-d', '--debug', dest='debug', action='store_true',
             help='log debugging output'
@@ -184,7 +200,16 @@ def setup_logging(name, args):
         '%(asctime)-6s: %(name)s - %(levelname)s - %(message)s'
     )
 
-    if logfile:
+    # No timestamp, used with syslog
+    simple_formatter = logging.Formatter(
+        '%(name)s - %(levelname)s - %(message)s'
+    )
+
+    if args.syslog and not args.nodaemon:
+        handler = logging.handlers.SysLogHandler(address=args.syslog_socket,
+                                                 facility=args.syslog_facility)
+        handler.setFormatter(simple_formatter)
+    elif logfile:
         handler = CompressedTimedRotatingFileHandler(
             logfile, when='D', interval=1, backupCount=7
         )
@@ -201,4 +226,10 @@ def setup_logging(name, args):
     elif args.verbose:
         logger.setLevel(level=logging.INFO)
 
+    if logfile and not args.syslog and args.user:
+        # NOTE(LinuxJedi): we are switching user so need to switch
+        # the ownership of the log file for rotation
+        os.chown(logger.handlers[0].baseFilename,
+                 pwd.getpwnam(args.user).pw_uid, -1)
+
     return logger

libra/mgm/mgm.py

@@ -22,7 +22,6 @@ import time
 import sys
 import os
 import threading
-import lockfile
 
 from novaclient import exceptions
 from libra.openstack.common import importutils
@@ -33,8 +32,7 @@ from libra.mgm.node_list import NodeList, AccessDenied
 
 
 class Server(object):
-    def __init__(self, logger, args):
+    def __init__(self, args):
-        self.logger = logger
         self.args = args
         self.ct = None
         self.ft = None
@@ -43,10 +41,12 @@ class Server(object):
         try:
             self.node_list = NodeList(self.args.datadir)
         except AccessDenied as exc:
-            self.logger.error(exc)
+            print(str(exc))
             self.shutdown(True)
 
     def main(self):
+        self.logger = setup_logging('libra_mgm', self.args)
+
         self.logger.info(
             'Libra Pool Manager started with a float of {nodes} nodes'
             .format(nodes=self.args.nodes)
@@ -429,46 +429,25 @@ def main():
         svr_list = args.api_server.split()
         args.api_server = svr_list
 
-    logger = setup_logging('libra_mgm', args)
+    server = Server(args)
-    server = Server(logger, args)
 
     if args.nodaemon:
         server.main()
     else:
         pidfile = daemon.pidfile.TimeoutPIDLockFile(args.pid, 10)
         if daemon.runner.is_pidfile_stale(pidfile):
-            logger.warning("Cleaning up stale PID file")
             pidfile.break_lock()
         context = daemon.DaemonContext(
             working_directory='/',
             umask=0o022,
-            pidfile=pidfile,
+            pidfile=pidfile
-            files_preserve=[logger.handlers[0].stream]
         )
         if args.user:
-            try:
+            context.uid = pwd.getpwnam(args.user).pw_uid
-                context.uid = pwd.getpwnam(args.user).pw_uid
-            except KeyError:
-                logger.critical("Invalid user: %s" % args.user)
-                return 1
-            # NOTE(LinuxJedi): we are switching user so need to switch
-            # the ownership of the log file for rotation
-            os.chown(logger.handlers[0].baseFilename, context.uid, -1)
         if args.group:
-            try:
+            context.gid = grp.getgrnam(args.group).gr_gid
-                context.gid = grp.getgrnam(args.group).gr_gid
-            except KeyError:
-                logger.critical("Invalid group: %s" % args.group)
-                return 1
-        try:
-            context.open()
-        except lockfile.LockTimeout:
-            logger.critical(
-                "Failed to lock pidfile %s, another instance running?",
-                args.pid
-            )
-            return 1
 
+        context.open()
         server.main()
 
     return 0

libra/statsd/main.py

@@ -15,9 +15,7 @@
 import daemon
 import daemon.pidfile
 import daemon.runner
-import lockfile
 import grp
-import os
 import pwd
 import time
 
@@ -27,9 +25,14 @@ from libra.statsd.drivers.base import known_drivers
 from libra.statsd.scheduler import Sched
 
 
-def start(logger, args, drivers):
+def start(args, drivers):
     """ Start the main server processing. """
 
+    logger = setup_logging('libra_statsd', args)
+
+    logger.info("Job server list: %s" % args.server)
+    logger.info("Selected drivers: {0}".format(args.driver))
+
     scheduler = Sched(logger, args, drivers)
     scheduler.start()
     while True:
@@ -82,8 +85,6 @@ def main():
 
     args = options.run()
 
-    logger = setup_logging('libra_statsd', args)
-
     if not args.server:
         # NOTE(shrews): Can't set a default in argparse method because the
         # value is appended to the specified default.
@@ -106,8 +107,6 @@ def main():
         svr_list = args.api_server.split()
         args.api_server = svr_list
 
-    logger.info("Job server list: %s" % args.server)
-    logger.info("Selected drivers: {0}".format(args.driver))
     if not isinstance(args.driver, list):
         args.driver = args.driver.split()
     for driver in args.driver:
@@ -116,39 +115,19 @@ def main():
         ))
 
     if args.nodaemon:
-        start(logger, args, drivers)
+        start(args, drivers)
     else:
         pidfile = daemon.pidfile.TimeoutPIDLockFile(args.pid, 10)
         if daemon.runner.is_pidfile_stale(pidfile):
-            logger.warning("Cleaning up stale PID file")
             pidfile.break_lock()
         context = daemon.DaemonContext(
             umask=0o022,
-            pidfile=pidfile,
+            pidfile=pidfile
-            files_preserve=[logger.handlers[0].stream]
         )
         if args.user:
-            try:
+            context.uid = pwd.getpwnam(args.user).pw_uid
-                context.uid = pwd.getpwnam(args.user).pw_uid
-            except KeyError:
-                logger.critical("Invalid user: %s" % args.user)
-                return 1
-            # NOTE(LinuxJedi): we are switching user so need to switch
-            # the ownership of the log file for rotation
-            os.chown(logger.handlers[0].baseFilename, context.uid, -1)
         if args.group:
-            try:
+            context.gid = grp.getgrnam(args.group).gr_gid
-                context.gid = grp.getgrnam(args.group).gr_gid
-            except KeyError:
-                logger.critical("Invalid group: %s" % args.group)
-                return 1
 
-        try:
+        context.open()
-            context.open()
+        start(args, drivers)
-        except lockfile.LockTimeout:
-            logger.critical(
-                "Failed to lock pidfile %s, another instance running?",
-                args.pid
-            )
-
-        start(logger, args, drivers)

libra/worker/main.py

@@ -16,8 +16,6 @@ import eventlet
 eventlet.monkey_patch()
 
 import daemon
-import lockfile
-import os
 import daemon.pidfile
 import daemon.runner
 import grp
@@ -37,10 +35,7 @@ class EventServer(object):
     non-daemon mode.
     """
 
-    def __init__(self, logger):
+    def main(self, args, tasks):
-        self.logger = logger
-
-    def main(self, tasks):
         """
         Main method of the server.
 
@@ -48,16 +43,22 @@ class EventServer(object):
             A tuple with two items: a function name, and a tuple with
             that function's arguments.
         """
-
         thread_list = []
+        logger = setup_logging('libra_worker', args)
 
-        for task, args in tasks:
+        logger.info("Selected driver: %s" % args.driver)
-            thread_list.append(eventlet.spawn(task, *args))
+        if args.driver == 'haproxy':
+            logger.info("Selected HAProxy service: %s" % args.haproxy_service)
+        logger.info("Job server list: %s" % args.server)
+
+        for task, task_args in tasks:
+            task_args = (logger,) + task_args  # Make the logger the first arg
+            thread_list.append(eventlet.spawn(task, *task_args))
 
         for thd in thread_list:
             thd.wait()
 
-        self.logger.info("Shutting down")
+        logger.info("Shutting down")
 
 
 def main():
@@ -90,8 +91,6 @@ def main():
     )
     args = options.run()
 
-    logger = setup_logging('libra_worker', args)
-
     if not args.server:
         # NOTE(shrews): Can't set a default in argparse method because the
         # value is appended to the specified default.
@@ -107,11 +106,9 @@ def main():
     # along to the Gearman task that will use it to communicate with
     # the device.
 
-    logger.info("Selected driver: %s" % args.driver)
     driver_class = importutils.import_class(known_drivers[args.driver])
 
     if args.driver == 'haproxy':
-        logger.info("Selected HAProxy service: %s" % args.haproxy_service)
         if args.user:
             user = args.user
         else:
@@ -127,52 +124,30 @@ def main():
     else:
         driver = driver_class()
 
-    logger.info("Job server list: %s" % args.server)
+    server = EventServer()
-    server = EventServer(logger)
 
     # Tasks to execute in parallel
     task_list = [
-        (config_thread, (logger, driver, args.server, args.reconnect_sleep))
+        (config_thread, (driver, args.server, args.reconnect_sleep))
     ]
 
     if args.nodaemon:
-        server.main(task_list)
+        server.main(args, task_list)
     else:
         pidfile = daemon.pidfile.TimeoutPIDLockFile(args.pid, 10)
         if daemon.runner.is_pidfile_stale(pidfile):
-            logger.warning("Cleaning up stale PID file")
             pidfile.break_lock()
         context = daemon.DaemonContext(
             working_directory='/etc/haproxy',
             umask=0o022,
-            pidfile=pidfile,
+            pidfile=pidfile
-            files_preserve=[logger.handlers[0].stream]
         )
         if args.user:
-            try:
+            context.uid = pwd.getpwnam(args.user).pw_uid
-                context.uid = pwd.getpwnam(args.user).pw_uid
-            except KeyError:
-                logger.critical("Invalid user: %s" % args.user)
-                return 1
-            # NOTE(LinuxJedi): we are switching user so need to switch
-            # the ownership of the log file for rotation
-            os.chown(logger.handlers[0].baseFilename, context.uid, -1)
         if args.group:
-            try:
+            context.gid = grp.getgrnam(args.group).gr_gid
-                context.gid = grp.getgrnam(args.group).gr_gid
-            except KeyError:
-                logger.critical("Invalid group: %s" % args.group)
-                return 1
 
-        try:
+        context.open()
-            context.open()
+        server.main(args, task_list)
-        except lockfile.LockTimeout:
-            logger.critical(
-                "Failed to lock pidfile %s, another instance running?",
-                args.pid
-            )
-            return 1
-
-        server.main(task_list)
 
     return 0