From afca2222b295326cd758f9e3cb31f241b6cc2e46 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Evrard <jean-philippe@evrard.me>
Date: Tue, 24 Apr 2018 21:36:21 +0000
Subject: [PATCH] Add Graylog Central Logging
This adds a central logging using graylog2 with little to no
code.
Change-Id: I63e59e249329ececf0598552b95329b38b4ed32c
---
graylog/README.rst | 119 ++++++++++++++++++++++++++
graylog/ansible-role-requirements.yml | 16 ++++
graylog/ansible.cfg | 2 +
graylog/graylog-forward-logs.yml | 44 ++++++++++
graylog/graylog2-install.yml | 76 ++++++++++++++++
5 files changed, 257 insertions(+)
create mode 100644 graylog/README.rst
create mode 100644 graylog/ansible-role-requirements.yml
create mode 100644 graylog/ansible.cfg
create mode 100644 graylog/graylog-forward-logs.yml
create mode 100644 graylog/graylog2-install.yml
diff --git a/graylog/README.rst b/graylog/README.rst
new file mode 100644
index 00000000..965bfee6
--- /dev/null
+++ b/graylog/README.rst
@@ -0,0 +1,119 @@
+Central Logging with Graylog2
+=============================
+
+Introduction
+------------
+
+This part of the ops repo is in charge of:
+
+* Setting up Graylog2 into the ``graylog_hosts`` group
+* Shipping all your hosts logs into Graylog2 using graylog native format (GELF)
+* Configuring haproxy for Graylog2
+
+Current limitations
+-------------------
+
+The upstream Graylog2 ansible role doesn't currently support deploying in a cluster
+setup, and therefore the deploy needs to be restricted to one backend for now:
+https://github.com/Graylog2/graylog-ansible-role/issues/89. It is all due to the
+fact the authentication sessions have to be shared on a mongoDB cluster, and no
+role is available to build the mongo cluster. Patches welcomed!
+
+Fetching the roles
+------------------
+
+To install Graylog2 you need to make sure all the necessary roles are in your environment,
+if you don't have them already.
+
+You can re-use the bootstrap-ansible script with this ansible-role-requirement file
+(see the OpenStack-Ansible reference documentation), or, simply run::
+
+ ansible-galaxy install -r ansible-role-requirements.yml
+
+
+Installing Graylog2 on graylog_hosts
+------------------------------------
+
+Add a file in /etc/openstack_deploy/user_graylog.yml, with the following content::
+
+ graylog_password_secret: "" # The output of `pwgen -N 1 -s 96`
+ graylog_root_username: "admin"
+ graylog_root_password_sha2: "" # The output of `echo -n yourpassword | shasum -a 256`
+ haproxy_extra_services:
+ - service:
+ haproxy_service_name: graylog
+ haproxy_backend_nodes: "{{ [groups['graylog_hosts'][0]] | default([]) }}"
+ haproxy_ssl: "{{ haproxy_ssl }}"
+ haproxy_port: 9000
+ haproxy_balance_type: http
+
+See more Graylog2 deploy variables in
+https://github.com/Graylog2/graylog-ansible-role/blob/e1159ec2712199f2da5768187cee84d1359bbd55/defaults/main.yml
+
+If you want the ``graylog_hosts`` group to match the existing ``log_hosts`` group,
+add the following in your ``/etc/openstack_deploy/inventory.ini``::
+
+ [graylog_hosts:children]
+ log_hosts
+
+To deploy Graylog2, simply run the install playbook::
+
+ openstack-ansible graylog2-install.yml
+
+To point haproxy to your new Graylog2 instance, re-run the ``haproxy-install.yml`` playbook.
+
+Note: If running Graylog2 on the same host as the load balancer, you'll hit an issue with an already
+taken port. In that case, either don't configure haproxy, or configure it to run on an interface not yet
+bound. For example, you can use the following line in your ``user_graylog.yml`` haproxy service section
+to bind only on the external lb vip address::
+
+ haproxy_bind: "{{ [external_lb_vip_address] }}"
+
+Note: You can optionally add a series of headers in your haproxy to help on the web interface
+redirection, if you have a specific network configuration.
+
+ http-request set-header X-Graylog-Server-URL https://{{ external_lb_vip_address }}:9000/api
+
+Configuration of Graylog2
+-------------------------
+
+Connect as the interface on your loadbalancer address, port 9000, with the user ``admin``, and the
+previously defined password whose shasum was given into ``graylog_root_password_sha2``.
+
+In the web interface, add the inputs you need.
+
+If you want to configure your your nodes with the provided playbook, you will need to
+create a new GELF UDP input on at least one of your Graylog2 nodes (select ``global`` if you want to
+listen on all the nodes).
+
+For the exercise, we are defining the port to listen to as UDP 12201.
+
+Sending logs to Graylog2
+------------------------
+
+Graylog2 can receive data with different protocols, but there is an efficient native format for it, GELF.
+
+All of this is configured in a single playbook: ``graylog-forward-logs.yml``.
+
+There are many packages to forward the journal into Graylog2, like the official `journal2gelf`_.
+The ``graylog-ship-logs.yml`` playbook uses a fork of `journal2gelf` using `gelfclient`_.
+It's lightweight and easy to install.
+
+This script needs to know where to forward to, and depends on how you configured Graylog2 at the
+previous step.
+
+In the example above, the following variables need to be set in
+``/etc/openstack_deploy/user_graylog.yml``::
+
+ graylog_targets:
+ - "{{ groups['graylog_hosts'][0] }}:12201"
+
+If you are shipping journals directly from containers to the host, there is no need to run this playbook
+on the full list of nodes. Instead, use the ansible ``--limit`` directive to restrict on which host
+this playbook should run.
+
+That's all folks!
+
+.. _journal2gelf: https://github.com/systemd/journal2gelf
+.. _gelfclient: https://github.com/nailgun/journal2gelf
+
diff --git a/graylog/ansible-role-requirements.yml b/graylog/ansible-role-requirements.yml
new file mode 100644
index 00000000..abca0149
--- /dev/null
+++ b/graylog/ansible-role-requirements.yml
@@ -0,0 +1,16 @@
+---
+- name: elastic.elasticsearch
+ src: https://github.com/elastic/ansible-elasticsearch.git
+ version: 3bdcd8fe4d0afdc2da5e12475b2093bb2bb3326b
+
+- name: jdauphant.nginx
+ src: https://github.com/jdauphant/ansible-role-nginx.git
+ version: 'v2.7.4'
+
+- name: geerlingguy.java
+ src: https://github.com/geerlingguy/ansible-role-java
+ version: ebe72b1b52fe0053bb156fd1b29d044f2048556b
+
+- name: Graylog2.graylog-ansible-role
+ src: https://github.com/Graylog2/graylog-ansible-role.git
+ version: e1159ec2712199f2da5768187cee84d1359bbd55
diff --git a/graylog/ansible.cfg b/graylog/ansible.cfg
new file mode 100644
index 00000000..ae190d32
--- /dev/null
+++ b/graylog/ansible.cfg
@@ -0,0 +1,2 @@
+[defaults]
+roles_path = /etc/ansible/roles
diff --git a/graylog/graylog-forward-logs.yml b/graylog/graylog-forward-logs.yml
new file mode 100644
index 00000000..ac787e94
--- /dev/null
+++ b/graylog/graylog-forward-logs.yml
@@ -0,0 +1,44 @@
+---
+- hosts: all:!log_hosts
+ gather_facts: no
+ vars:
+ graylog_forwarder_system_packages:
+ - python-systemd
+ graylog_forwarder_pip_packages:
+ - gelfclient==0.0.7
+ - journal2gelf==2.0.0
+ tasks:
+ #- name: Gather variables for each operating system
+ # include_vars: "{{ item }}"
+ # with_first_found:
+ # - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
+ # - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
+ # - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
+ # - "{{ ansible_distribution | lower }}.yml"
+ # - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
+ # - "{{ ansible_os_family | lower }}.yml"
+ # tags:
+ # - always
+
+ - name: Install graylog forwarder package requirements
+ package:
+ name: "{{ graylog_forwarder_system_packages }}"
+ state: present
+ # Graylog wasn't build in repo, and requires running "isolated"
+ - name: Install graylog forwarder requirements
+ pip:
+ name: "{{ graylog_forwarder_pip_packages }}"
+ state: present
+ - name: Install the log forwarder service.
+ include_role:
+ name: systemd_service
+ with_items: "{{ graylog_targets }}"
+ loop_control:
+ loop_var: graylog_target
+ vars:
+ systemd_services:
+ - service_name: gelf-forwarder
+ state: started
+ execstarts:
+ - "/usr/local/bin/journal2gelf {{ graylog_target }}"
+
diff --git a/graylog/graylog2-install.yml b/graylog/graylog2-install.yml
new file mode 100644
index 00000000..b6384fc4
--- /dev/null
+++ b/graylog/graylog2-install.yml
@@ -0,0 +1,76 @@
+---
+- name: Ensure sysctl
+ hosts: graylog_hosts
+ gather_facts: true
+ tasks:
+ - name: Setup sysctl
+ sysctl:
+ name: vm.max_map_count
+ value: "262144"
+ state: present
+ delegate_to: "{{ physical_host }}"
+
+- name: Install java from openjdk
+ hosts: graylog_hosts
+ tasks:
+ # TODO: Replace this with a group var to log_hosts to use openstack_hosts role.
+ - name: installing repo for Java 8 in Ubuntu 16.04
+ apt_repository: repo='ppa:openjdk-r/ppa'
+ when: ansible_distribution_release == 'xenial'
+
+ - name: Install Java for Ubuntu
+ include_role:
+ name: geerlingguy.java
+ when:
+ - ansible_os_family | lower == 'debian'
+ - ansible_distribution_release == 'xenial'
+ vars:
+ java_packages:
+ - openjdk-8-jdk
+
+ # TODO: Add SUSE support
+ - name: Install Java on CentOS
+ package:
+ name: java-1.8.0-openjdk-headless.x86_64
+ state: present
+ when: ansible_os_family | lower == 'redhat'
+
+- name: Install graylog
+ hosts: graylog_hosts
+ vars:
+ # Graylog is compatible with elasticsearch 5.x since version 2.3.0, so ensure to use the right combination for your installation
+ # Also use the right branch of the Elasticsearch Ansible role, master supports 5.x.
+ es_java_install: False
+ es_java: openjdk-8-jre-headless
+ es_major_version: "5.x"
+ es_version: "5.6.7"
+ es_instance_name: 'graylog'
+ es_scripts: False
+ es_templates: False
+ es_version_lock: False
+ es_heap_size: 1g
+ es_config: {
+ node.name: "graylog",
+ cluster.name: "graylog",
+ http.port: 9200,
+ transport.tcp.port: 9300,
+ network.host: "{{ es_bind_address | default('0.0.0.0') }}",
+ node.data: true,
+ node.master: "{{ inventory_hostname == groups['graylog_hosts'][0] }}",
+ }
+
+ graylog_install_java: False
+ graylog_install_mongodb: True
+ graylog_install_nginx: False #Will be behind your LB
+ graylog_web_endpoint_uri: "https://{{ external_lb_vip_address }}:9000/api/"
+ graylog_is_master: "{{ inventory_hostname == groups['graylog_hosts'][0] }}"
+ graylog_elasticsearch_hosts: "{{ groups['graylog_hosts'] | map('extract', hostvars, 'ansible_host') | map('regex_replace', '^(.*)$', 'http://\\1:9200') | join(', ') }}"
+ graylog_web_listen_uri: "http://{{ ansible_host }}:9000/"
+ graylog_rest_listen_uri: "http://{{ ansible_host }}:9000/api/"
+ # TODO(evrardjp): Replace this with a proper test when
+ # https://github.com/Graylog2/graylog-ansible-role/pull/88 has merged
+ graylog_not_testing: False
+ roles:
+ # TODO: Contribute to the role for SUSE support
+ - role: Graylog2.graylog-ansible-role
+ tags: graylog