Feature: Keep alive interval of 60 by default

In situations where the Gerrit server dies suddenly (not a clean
process or system shutdown, but loss of networking or an underlying
hypervisor crash), the TCP socket for the stream-events feed can be
left indefinitely hung since the client has no reason to send new
messages and will never notice that there are no further messages
arriving from the server.

Gerritlib exposes a keep alive option for Gerrit stream-events
connections which can double as a sort of dead peer detection, but
it defaults to 0 (off). Set this to 60 seconds, while allowing it to
be configured to an arbitrary value in case this is unsuitable.

Change-Id: Ib92f1ee819f060dc52163735fe8d87d6d82ab788

doc/source/installation.rst

@@ -26,6 +26,7 @@ when starting the bot. It should look like::
   key=/path/to/id_rsa
   host=review.example.com
   port=29418
+  keep_alive=300  # (optional value in seconds, defaults to 60, 0 to disable)
 
 The second, referenced by ``[ircbot]channel_config`` in the above, configures
 the IRC channels and the events and projects that each channel is interested

gerritbot/bot.py

@@ -59,6 +59,7 @@ user=gerrit2
 key=/path/to/id_rsa
 host=review.example.com
 port=29418
+keep_alive=300  # (optional value in seconds, defaults to 60, 0 to disable)
 
 [mqtt]
 host=example.com
@@ -190,7 +191,7 @@ class SASLGerritBot(SASL, BaseGerritBot):
 
 class Gerrit(threading.Thread):
     def __init__(self, ircbot, channel_config, server,
-                 username, port=29418, keyfile=None):
+                 username, port=29418, keyfile=None, keep_alive=60):
         super(Gerrit, self).__init__()
         self.ircbot = ircbot
         self.channel_config = channel_config
@@ -199,6 +200,7 @@ class Gerrit(threading.Thread):
         self.username = username
         self.port = port
         self.keyfile = keyfile
+        self.keep_alive = keep_alive
         self.connected = False
 
     def connect(self):
@@ -206,7 +208,8 @@ class Gerrit(threading.Thread):
         import gerritlib.gerrit
         try:
             self.gerrit = gerritlib.gerrit.Gerrit(
-                self.server, self.username, self.port, self.keyfile)
+                self.server, self.username, self.port, self.keyfile,
+                keep_alive_interval=self.keep_alive)
             self.gerrit.startWatching()
             self.log.info('Start watching Gerrit event stream.')
             self.connected = True
@@ -536,12 +539,17 @@ def _main(config):
                        config.getint('mqtt', 'port'),
                        config.getboolean('mqtt', 'websocket'))
     else:
+        if config.has_option('gerrit', 'keep_alive'):
+            keep_alive = config.getint('gerrit', 'keep_alive')
+        else:
+            keep_alive = 60
         g = Gerrit(bot,
                    channel_config,
                    config.get('gerrit', 'host'),
                    config.get('gerrit', 'user'),
                    config.getint('gerrit', 'port'),
-                   config.get('gerrit', 'key'))
+                   config.get('gerrit', 'key'),
+                   keep_alive=keep_alive)
     g.start()
     bot.start()
 

requirements.txt

@@ -1,4 +1,4 @@
-gerritlib
+gerritlib>=0.10.0
 irc==18.0.0
 pyyaml
 python-daemon