Add logstash forwarder to agent.

Change-Id: I6ba34cadaf9f7d69d6b7039b9ebea9523d433fd1

install/install.sh

@@ -343,6 +343,9 @@ echo "Install the Package Installer Tool"
 # source ${COMPASSDIR}/install/chef.sh || exit $?
 source ${COMPASSDIR}/install/ansible.sh || exit $?
 
+echo "Install log agent."
+source ${COMPASSDIR}/install/logstash-forwarder.sh || exit $?
+
 if [ "$FULL_COMPASS_SERVER" == "true"]; then
     echo "Download and install Compass Web"
     source ${COMPASSDIR}/install/compass_web.sh || exit $?

install/logstash-forwarder.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+
+echo "Installing logstash-forwarder"
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+source $DIR/install.conf
+
+sudo cp -rf $COMPASSDIR/misc/logstash-forwarder/logstash-forwarder.repo /etc/yum.repos.d/logstash-forwarder.repo
+sudo yum -y install logstash-forwarder
+sudo rm -rf /etc/logstash-forwarder.conf
+sudo cp -rf $COMPASSDIR/misc/logstash-forwarder/logstash-forwarder.conf /etc/logstash-forwarder.conf
+sudo mkdir -p /etc/pki/tls/certs
+sudo cp -rf $COMPASSDIR/misc/logstash-forwarder/logstash-forwarder.crt /etc/pki/tls/certs/logstash-forwarder.crt
+
+sudo systemctl restart logstash-forwarder.service
+sleep 3
+echo "checking if logstash-forwarder is running"
+sudo systemctl status logstash-forwarder.service
+if [[ "$?" != 0 ]]; then
+    echo "logstash-forwarder is not running"
+    exit
+fi

misc/logstash-forwarder/logstash-forwarder.conf

@@ -0,0 +1,57 @@
+{
+  # The network section covers network configuration :)
+  "network": {
+    # A list of downstream servers listening for our messages.
+    # logstash-forwarder will pick one at random and only switch if
+    # the selected one appears to be dead or unresponsive
+    "servers": [ "www.stack360.io:5000" ],
+
+    # The path to your client ssl certificate (optional)
+    #"ssl certificate": "./logstash-forwarder.crt",
+    # The path to your client ssl key (optional)
+    #"ssl key": "./logstash-forwarder.key",
+
+    # The path to your trusted ssl CA file. This is used
+    # to authenticate your downstream server.
+    "ssl ca": "/etc/pki/tls/certs/logstash-forwarder.crt",
+
+    # Network timeout in seconds. This is most important for
+    # logstash-forwarder determining whether to stop waiting for an
+    # acknowledgement from the downstream server. If an timeout is reached,
+    # logstash-forwarder will assume the connection or server is bad and
+    # will connect to a server chosen at random from the servers list.
+    "timeout": 15
+  },
+
+  # The list of files configurations
+  "files": [
+    {
+        "paths": [
+            "/var/log/compass/celery.log"
+        ]
+    }
+    # An array of hashes. Each hash tells what paths to watch and
+    # what fields to annotate on events from those paths.
+    #{
+      #"paths": [
+        # single paths are fine
+        #"/var/log/messages",
+        # globs are fine too, they will be periodically evaluated
+        # to see if any new files match the wildcard.
+        #"/var/log/*.log"
+      #],
+
+      # A dictionary of fields to annotate on each event.
+      #"fields": { "type": "syslog" }
+    #}, {
+      # A path of "-" means stdin.
+      #"paths": [ "-" ],
+      #"fields": { "type": "stdin" }
+    #}, {
+      #"paths": [
+        #"/var/log/apache/httpd-*.log"
+      #],
+      #"fields": { "type": "apache" }
+    #}
+  ]
+}

misc/logstash-forwarder/logstash-forwarder.crt

@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFAzCCAuugAwIBAgIJAKrryFncVfJNMA0GCSqGSIb3DQEBCwUAMBgxFjAUBgNV
+BAMMDSouc3RhY2szNjAuaW8wHhcNMTYwODE4MTcwNzEyWhcNMjYwODE2MTcwNzEy
+WjAYMRYwFAYDVQQDDA0qLnN0YWNrMzYwLmlvMIICIjANBgkqhkiG9w0BAQEFAAOC
+Ag8AMIICCgKCAgEAw4p1OVw8tMeYfk9828FiRLPhYWwHD2OCVwlKr8P3bl974I/P
+PhYTkrjEEe5SDYCWNcO58MxJ5X1vB2uQGNBBUFMni+KOqbVvcbPLL4Mkf8pjLdzD
+2pItE1X7UQ8p1iYBEyAnLoV4MjA7CJ4MmUtOOcCb6keHWEhyJFOj9IzkTjFCbKuL
+Y0paxayQOFlkuEK2d3Aa2HXgTLh3FFUX3kRqOcGg1uxEs2awf0nwP/NwcyfAgTSe
+6yqVjnb3GLYdy283FqvxW1MrZS9UAxp/tAXRgMT8I9L2wSrjnPmrabBv5I+Kuf0p
+EmLmxedOCTQsRYtYGL7TmySYgGuYtt/4UNKaG9tCPBnRXuMzlRClBDIoDhwLEKzC
+LlE8JKyClNgutMKcrHbWlEJiGxooyDC1H9gwkHTi3w7qF9BYYBhkXFN2Sl0mPk0m
+6NMfhQUAeqI0HlOsAX/HLwSWyRl0Nr2rvVJqgbbHRF18pADitQEuc/koT9qhON5f
+BReYhQZIarwPJ/UBgwgadrP79bvWJ5u9Oga6H7yHf49/UYD3gQCvu3/Hxo/IY6AX
+86eZg1ZObD6J0xpWb5jskVSgugar6Xq/h6iRkX8O3ssUdIyIxsIMhtCgxykSmdQY
+FDEIWGZq2kdEVvF6GR/EYJRxvASI+27TXzuxP1UodQQOXa8xySkKu9U5SqMCAwEA
+AaNQME4wHQYDVR0OBBYEFPLilPT6k6rRlxv9kCTW38VYy5AuMB8GA1UdIwQYMBaA
+FPLilPT6k6rRlxv9kCTW38VYy5AuMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL
+BQADggIBAAdDy+R0bGCBQgiF2fGunl+AZ8zd0MZbtL4Kv53gunlhyaPQJ3Z+e3GF
+V+Z9BhvMUXE/XN3bsAIZYJekvclysYLBdK1C8n9Rli+AbBSGjwgttRAXeEqaZCbE
+QrNPukRgHThv6hyJNcf1TnR70xCBlcYOGQkEqWx1g0xrsG8ryGbum0BAG0YWLCYq
+BboP16FGAPjDlb10ysWy7HuGJorf470Kyb2iRfp4PX/zdYbcA8gcggGCT323JwuD
+Qwnd8kyX6+6pNhBJE3gAyPDhqvbFbpmo/Ia5pqmJkL3APee1bVI2itqZB/HnmyZ5
+UCIzqf4uAuEodN6yXiImHt0TagXtL4eEXNE2qjwsFSaHeyP6iXPX5tc8RMHzFQjH
+I3MzROHaeVUxs2rMxETGvq4+DebhGzCBKaJUWfMV6Y1+ovrE0MowcR8nO1Q4YQt3
+to+W5IrjI7zzQ7+4XqZm+Yz1DQ4Kr7s2iyAExvkq7kU5FAow6SLPIaOl7kbas2M4
+fwFisuEQT2Om+hbWWZTJ1T45KU5NjznkPqJZ9dCdyqs2mH7BE4vOkOULq81uFG06
+VnJHcO+wZM1iCLa1hy7F2S4fDTjTBYDcuD5GFkulFLeFH5X7zoPz20OGTRMXZONI
+CHEk1ibp8j/Q6bw1zd0jGm3KDUSx+0/Avfve/e28U4KAdNekrgQf
+-----END CERTIFICATE-----

misc/logstash-forwarder/logstash-forwarder.repo

@@ -0,0 +1,6 @@
+[logstashforwarder]
+name=logstashforwarder repository
+baseurl=http://packages.elasticsearch.org/logstashforwarder/centos
+gpgcheck=1
+gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch
+enabled=1