Update README, beat deployment, and configs

Beat set is using a loop to ship both templates and dashboards using
different commands. This is being done to ensure no data is lost or
mis-setup during index creation.

Packetbeat only needs to be installed on hosts

All of the beats are now using standard field templates.

Change-Id: Ie6220eb94a12780ad122ba367bf7654d97c212e8
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/installAuditbeat.yml

@@ -26,6 +26,14 @@
         src: templates/auditbeat.yml.j2
         dest: /etc/auditbeat/auditbeat.yml
 
+    - name: Stop auditd
+      systemd:
+        name: "auditd"
+        enabled: "{{ not inventory_hostname in groups['kibana'] | default([]) }}"
+        state: stopped
+      when:
+        - not apply_security_hardening | default(true) | bool
+
     - name: Enable and restart auditbeat
       systemd:
         name: "auditbeat"
@@ -48,6 +56,10 @@
         {% endfor %}
         {% set elasticsearch_hosts = [IP_ARR | map('regex_replace', '$', ':' ~ elastic_port|string()) | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' )] %}
         auditbeat setup
+        {{ item }}
         -E 'output.logstash.enabled=false'
         -E 'output.elasticsearch.hosts={{ elasticsearch_hosts }}'
         -e -v
+      with_items:
+        - "--template"
+        - "--dashboards"

elk_metrics_6x/installMetricbeat.yml

@@ -72,6 +72,10 @@
         {% endfor %}
         {% set elasticsearch_hosts = [IP_ARR | map('regex_replace', '$', ':' ~ elastic_port|string()) | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' )] %}
         metricbeat setup
+        {{ item }}
         -E 'output.logstash.enabled=false'
         -E 'output.elasticsearch.hosts={{ elasticsearch_hosts }}'
         -e -v
+      with_items:
+        - "--template"
+        - "--dashboards"

elk_metrics_6x/installPacketbeat.yml

@@ -1,6 +1,6 @@
 ---
 - name: Install Packetbeat
-  hosts: all
+  hosts: hosts
   become: true
   vars:
     haproxy_ssl: false
@@ -13,9 +13,12 @@
 
     - name: Ensure packetbeat is installed
       apt:
-        name: packetbeat
+        name: "{{ item }}"
         state: present
         update_cache: true
+      with_items:
+        - tcpdump
+        - packetbeat
 
   post_tasks:
     - name: Drop packetbeat conf file
@@ -31,7 +34,7 @@
 
 
 - name: Load Packetbeat Dashboards
-  hosts: all[0]
+  hosts: hosts[0]
   become: true
   vars_files:
     - vars/variables.yml
@@ -45,6 +48,10 @@
         {% endfor %}
         {% set elasticsearch_hosts = [IP_ARR | map('regex_replace', '$', ':' ~ elastic_port|string()) | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' )] %}
         packetbeat setup
+        {{ item }}
         -E 'output.logstash.enabled=false'
         -E 'output.elasticsearch.hosts={{ elasticsearch_hosts }}'
         -e -v
+      with_items:
+        - "--template"
+        - "--dashboards"

elk_metrics_6x/readme.rst

@@ -1,17 +1,12 @@
-install Elk stack with topbeat to gather metrics
+Install ELK with beats to gather metrics
-#################################################
+########################################
 :tags: openstack, ansible
 
-
-Changelog
----------
-2018-03-06 Per Abildgaard Toft (per@minfejl.dk): Updated to version Elasticsearch,Logstash and Kibana 6.x. Changed Topebeat (deprecated) to metricbeat. Included haproxy endpoint configuration.
-
-
 About this repository
 ---------------------
 
-This set of playbooks will deploy elk cluster (Elasticsearch, Logstash, Kibana) with topbeat to gather metrics from hosts metrics to the ELK cluster.
+This set of playbooks will deploy elk cluster (Elasticsearch, Logstash, Kibana)
+with topbeat to gather metrics from hosts metrics to the ELK cluster.
 
 Process
 -------
@@ -27,7 +22,7 @@ Copy the env.d file into place
 
 .. code-block:: bash
 
-    cd openstack-ansible-ops/elk_metrics_6x
+    cd /opt/openstack-ansible-ops/elk_metrics_6x
     cp env.d/elk.yml /etc/openstack_deploy/env.d/
 
 Copy the conf.d file into place
@@ -36,7 +31,9 @@ Copy the conf.d file into place
 
     cp conf.d/elk.yml /etc/openstack_deploy/conf.d/
 
-In **elk.yml**, list your logging hosts under elastic-logstash_hosts to create the elasticsearch cluster in multiple containers and one logging host under kibana_hosts to create the kibana container
+In **elk.yml**, list your logging hosts under elastic-logstash_hosts to create
+the elasticsearch cluster in multiple containers and one logging host under
+kibana_hosts to create the kibana container
 
 .. code-block:: bash
 
@@ -70,7 +67,8 @@ Install Kibana, nginx reverse proxy and metricbeat on the kibana container
     cd /opt/openstack-ansible-ops/elk_metrics_6x
     openstack-ansible installKibana.yml
 
-install Metricbeat everywhere to start shipping metrics to our logstash instances
+Install Metricbeat everywhere to start shipping metrics to our logstash
+instances
 
 .. code-block:: bash
 
@@ -79,7 +77,8 @@ install Metricbeat everywhere to start shipping metrics to our logstash instance
 
 Optional | conigure haproxy endpoints
 
-Edit the `/etc/openstack_deploy/user_variables.yml` file and add fiel following lines
+Edit the `/etc/openstack_deploy/user_variables.yml` file and add fiel following
+lines
 
 .. code-block:: yaml
 

elk_metrics_6x/templates/auditbeat.yml.j2

@@ -32,7 +32,7 @@ auditbeat.modules:
 # The auditd module collects events from the audit framework in the Linux
 # kernel. You need to specify audit rules for the events that you want to audit.
 - module: auditd
-  socket_type: multicast
+  socket_type: {{ (apply_security_hardening | default(true) | bool) | ternary('multicast', 'unicast') }}
   resolve_ids: true
   failure_mode: silent
   backlog_limit: 8196
@@ -55,14 +55,18 @@ auditbeat.modules:
     ## Executions.
     -a always,exit -F arch=b64 -S execve,execveat -k exec
 
-    ## Identity changes.
+    # Things that affect identity.
     -w /etc/group -p wa -k identity
     -w /etc/passwd -p wa -k identity
     -w /etc/gshadow -p wa -k identity
+    -w /etc/shadow -p wa -k identity
+
+    # Unauthorized access attempts to files (unsuccessful).
+    -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -F auid>=1000 -F auid!=4294967295 -F key=access
+    -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -F auid>=1000 -F auid!=4294967295 -F key=access
+    -a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EACCES -F auid>=1000 -F auid!=4294967295 -F key=access
+    -a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EPERM -F auid>=1000 -F auid!=4294967295 -F key=access
 
-    ## Unauthorized access attempts.
-    -a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access
-    -a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access
 {% endif %}
 
 # The file integrity module sends events when files are changed (created,
@@ -71,11 +75,12 @@ auditbeat.modules:
   paths:
   - /bin
   - /etc/apt
-  - /etc/networking
+  - /etc/network
   - /etc/openstack_deploy
   - /etc/sysconfig
   - /etc/systemd
   - /etc/yum
+  - /etc/zypp
   - /openstack/venvs
   - /sbin
   - /usr/bin
@@ -96,11 +101,11 @@ auditbeat.modules:
 
   # Average scan rate. This throttles the amount of CPU and I/O that Auditbeat
   # consumes at startup while scanning. Default is "50 MiB".
-  scan_rate_per_sec: 50 MiB
+  scan_rate_per_sec: 64 MiB
 
   # Limit on the size of files that will be hashed. Default is "100 MiB".
   # Limit on the size of files that will be hashed. Default is "100 MiB".
-  max_file_size: 100 MiB
+  max_file_size: 128 MiB
 
   # Hash types to compute when the file changes. Supported types are
   # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384,
@@ -774,15 +779,15 @@ setup.template.enabled: true
 
 # Template name. By default the template name is "auditbeat-%{[beat.version]}"
 # The template name and pattern has to be set in case the elasticsearch index pattern is modified.
-#setup.template.name: "auditbeat-%{[beat.version]}"
+setup.template.name: "auditbeat-%{[beat.version]}"
 
 # Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings.
 # The first part is the version of the beat and then -* is used to match all daily indices.
 # The template name and pattern has to be set in case the elasticsearch index pattern is modified.
-#setup.template.pattern: "auditbeat-%{[beat.version]}-*"
+setup.template.pattern: "auditbeat-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
-#setup.template.fields: "${path.config}/fields.yml"
+setup.template.fields: "${path.config}/fields.yml"
 
 # Overwrite existing template
 setup.template.overwrite: true

elk_metrics_6x/templates/metricbeat.yml.j2

@@ -79,21 +79,21 @@ metricbeat.modules:
   # in the top N by CPU or memory, in order to reduce the number of documents created.
   # If both the `by_cpu` and `by_memory` options are used, the union of the two sets
   # is included.
-  #process.include_top_n:
+  process.include_top_n:
     #
     # Set to false to disable this feature and include all processes
-    #enabled: true
+    enabled: true
 
     # How many processes to include from the top by CPU. The processes are sorted
     # by the `system.process.cpu.total.pct` field.
-    #by_cpu: 0
+    by_cpu: 20
 
     # How many processes to include from the top by memory. The processes are sorted
     # by the `system.process.memory.rss.bytes` field.
-    #by_memory: 0
+    by_memory: 20
 
   # If false, cmdline of a process is not cached.
-  #process.cmdline.cache.enabled: true
+  process.cmdline.cache.enabled: true
 
   # Enable collection of cgroup metrics from processes on Linux.
   process.cgroups.enabled: true
@@ -107,9 +107,9 @@ metricbeat.modules:
   process.include_cpu_ticks: {{ inventory_hostname == physical_host }}
 
   # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
-  #socket.reverse_lookup.enabled: false
+  socket.reverse_lookup.enabled: true
-  #socket.reverse_lookup.success_ttl: 60s
+  socket.reverse_lookup.success_ttl: 60s
-  #socket.reverse_lookup.failure_ttl: 60s
+  socket.reverse_lookup.failure_ttl: 60s
 
 ##------------------------------ Aerospike Module -----------------------------
 #- module: aerospike
@@ -1165,15 +1165,15 @@ setup.template.enabled: true
 
 # Template name. By default the template name is "metricbeat-%{[beat.version]}"
 # The template name and pattern has to be set in case the elasticsearch index pattern is modified.
-#setup.template.name: "metricbeat-%{[beat.version]}"
+setup.template.name: "metricbeat-%{[beat.version]}"
 
 # Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings.
 # The first part is the version of the beat and then -* is used to match all daily indices.
 # The template name and pattern has to be set in case the elasticsearch index pattern is modified.
-#setup.template.pattern: "metricbeat-%{[beat.version]}-*"
+setup.template.pattern: "metricbeat-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
-#setup.template.fields: "${path.config}/fields.yml"
+setup.template.fields: "${path.config}/fields.yml"
 
 # Overwrite existing template
 setup.template.overwrite: true

elk_metrics_6x/templates/packetbeat.yml.j2

@@ -18,24 +18,24 @@ packetbeat.interfaces.device: any
 # not the fastest option.
 # * af_packet, which uses memory-mapped sniffing. This option is faster than
 # libpcap and doesn't require a kernel module, but it's Linux-specific.
-#packetbeat.interfaces.type: pcap
+packetbeat.interfaces.type: af_packet
 
 # The maximum size of the packets to capture. The default is 65535, which is
 # large enough for almost all networks and interface types. If you sniff on a
 # physical network interface, the optimal setting is the MTU size. On virtual
 # interfaces, however, it's safer to accept the default value.
-#packetbeat.interfaces.snaplen: 65535
+packetbeat.interfaces.snaplen: 65535
 
 # The maximum size of the shared memory buffer to use between the kernel and
 # user space. A bigger buffer usually results in lower CPU usage, but consumes
 # more memory. This setting is only available for the af_packet sniffer type.
 # The default is 30 MB.
-#packetbeat.interfaces.buffer_size_mb: 30
+packetbeat.interfaces.buffer_size_mb: 30
 
 # Packetbeat automatically generates a BPF for capturing only the traffic on
 # ports where it expects to find known protocols. Use this settings to tell
 # Packetbeat to generate a BPF filter that accepts VLAN tags.
-#packetbeat.interfaces.with_vlans: true
+packetbeat.interfaces.with_vlans: true
 
 # Use this setting to override the automatically generated BPF filter.
 #packetbeat.interfaces.bpf_filter:
@@ -44,7 +44,7 @@ packetbeat.interfaces.device: any
 
 packetbeat.flows:
   # Enable Network flows. Default: true
-  #enabled: true
+  enabled: true
 
   # Set network flow timeout. Flow is killed if no packet is received before being
   # timed out.
@@ -60,10 +60,17 @@ packetbeat.protocols:
   # Enable ICMPv4 and ICMPv6 monitoring. Default: true
   enabled: true
 
-{% if inventory_hostname in groups['rabbitmq_all'] | default([]) %}
 - type: amqp
   # Enable AMQP monitoring. Default: true
-  enabled: true
+{% set ns = namespace(enabled=(inventory_hostname in groups['rabbitmq_all'] | default([]))) %}
+{% if not ns.enabled | bool %}
+{%   for _item in groups['rabbitmq_all'] | default([]) %}
+{%     if not ns.enabled | bool | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
+{%       set ns.enabled = true %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+  enabled: {{ ns.enabled | bool }}
 
   # Configure the ports where to listen for AMQP traffic. You can disable
   # the AMQP protocol by commenting out the list of ports.
@@ -97,9 +104,11 @@ packetbeat.protocols:
   # Transaction timeout. Expired transactions will no longer be correlated to
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
-{% endif %}
 
 - type: cassandra
+  # Enable cassandra monitoring. Default: false
+  enabled: false
+
   #Cassandra port for traffic monitoring.
   ports: [9042]
 
@@ -156,10 +165,17 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
-{% if inventory_hostname in groups['shared-infra_hosts'] | default([]) %}
 - type: http
   # Enable HTTP monitoring. Default: true
-  enabled: true
+{% set ns = namespace(enabled=(inventory_hostname in groups['shared-infra_hosts'] | default([]))) %}
+{% if not ns.enabled | bool %}
+{%   for _item in groups['shared-infra_hosts'] | default([]) %}
+{%     if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
+{%       set ns.enabled = true %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+  enabled: {{ ns.enabled }}
 
   # Configure the ports where to listen for HTTP traffic. You can disable
   # the HTTP protocol by commenting out the list of ports.
@@ -209,12 +225,18 @@ packetbeat.protocols:
   # Maximum message size. If an HTTP message is larger than this, it will
   # be trimmed to this size. Default is 10 MB.
   #max_message_size: 10485760
-{% endif %}
 
-{% if inventory_hostname in groups['memcached_all'] | default([]) %}
 - type: memcache
   # Enable memcache monitoring. Default: true
-  enabled: true
+{% set ns = namespace(enabled=(inventory_hostname in groups['memcached_all'] | default([]))) %}
+{% if not ns.enabled | bool %}
+{%   for _item in groups['memcached_all'] | default([]) %}
+{%     if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
+{%       set ns.enabled = true %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+  enabled: {{ ns.enabled }}
 
   # Configure the ports where to listen for memcache traffic. You can disable
   # the Memcache protocol by commenting out the list of ports.
@@ -260,12 +282,18 @@ packetbeat.protocols:
   # Transaction timeout. Expired transactions will no longer be correlated to
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
-{% endif %}
 
-{% if inventory_hostname in groups['galera_all'] | default([]) %}
 - type: mysql
   # Enable mysql monitoring. Default: true
-  enabled: true
+{% set ns = namespace(enabled=(inventory_hostname in groups['galera_all'] | default([]))) %}
+{% if not ns.enabled | bool %}
+{%   for _item in groups['galera_all'] | default([]) %}
+{%     if not ns.enabled | bool and _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
+{%       set ns.enabled = true %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+  enabled: {{ ns.enabled }}
 
   # Configure the ports where to listen for MySQL traffic. You can disable
   # the MySQL protocol by commenting out the list of ports.
@@ -282,11 +310,10 @@ packetbeat.protocols:
   # Transaction timeout. Expired transactions will no longer be correlated to
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
-{% endif %}
 
 - type: pgsql
   # Enable pgsql monitoring. Default: true
-  #enabled: true
+  enabled: false
 
   # Configure the ports where to listen for Pgsql traffic. You can disable
   # the Pgsql protocol by commenting out the list of ports.
@@ -306,7 +333,7 @@ packetbeat.protocols:
 
 - type: redis
   # Enable redis monitoring. Default: true
-  #enabled: true
+  enabled: false
 
   # Configure the ports where to listen for Redis traffic. You can disable
   # the Redis protocol by commenting out the list of ports.
@@ -326,7 +353,7 @@ packetbeat.protocols:
 
 - type: thrift
   # Enable thrift monitoring. Default: true
-  #enabled: true
+  enabled: false
 
   # Configure the ports where to listen for Thrift-RPC traffic. You can disable
   # the Thrift-RPC protocol by commenting out the list of ports.
@@ -381,7 +408,7 @@ packetbeat.protocols:
 
 - type: mongodb
   # Enable mongodb monitoring. Default: true
-  #enabled: true
+  enabled: false
 
   # Configure the ports where to listen for MongoDB traffic. You can disable
   # the MongoDB protocol by commenting out the list of ports.
@@ -409,10 +436,17 @@ packetbeat.protocols:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
-{% if (inventory_hostname in groups['glance_all'] | default([])) or (inventory_hostname in groups['nova_compute'] | default([])) %}
 - type: nfs
   # Enable NFS monitoring. Default: true
-  enabled: true
+{% set ns = namespace(enabled=((inventory_hostname in groups['glance_all'] | default([])) or (inventory_hostname in groups['nova_compute'] | default([])))) %}
+{% if not ns.enabled | bool %}
+{%   for _item in groups['glance_all'] | default([]) + groups['nova_compute'] | default([]) %}
+{%     if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
+{%       set ns.enabled = true %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+  enabled: {{ ns.enabled }}
 
   # Configure the ports where to listen for NFS traffic. You can disable
   # the NFS protocol by commenting out the list of ports.
@@ -429,12 +463,18 @@ packetbeat.protocols:
   # Transaction timeout. Expired transactions will no longer be correlated to
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
-{% endif %}
 
-{% if (inventory_hostname in groups['haproxy_all'] | default([])) or (inventory_hostname in groups['horizon_alll'] | default([])) %}
 - type: tls
   # Enable TLS monitoring. Default: true
-  #enabled: true
+{% set ns = namespace(enabled=((inventory_hostname in groups['haproxy_all'] | default([])) or (inventory_hostname in groups['horizon_all'] | default([])))) %}
+{% if not ns.enabled | bool %}
+{%   for _item in groups['haproxy_all'] | default([]) + groups['horizon_all'] | default([]) %}
+{%     if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
+{%       set ns.enabled = true %}
+{%     endif %}
+{%   endfor %}
+{% endif %}
+  enabled: {{ ns.enabled }}
 
   # Configure the ports where to listen for TLS traffic. You can disable
   # the TLS protocol by commenting out the list of ports.
@@ -447,7 +487,6 @@ packetbeat.protocols:
   # If this option is enabled, the raw certificates will be stored
   # in PEM format under the `raw` key. The default is false.
   #include_raw_certificates: false
-{% endif %}
 
 #=========================== Monitored processes ==============================
 
@@ -1127,15 +1166,15 @@ setup.template.enabled: true
 
 # Template name. By default the template name is "packetbeat-%{[beat.version]}"
 # The template name and pattern has to be set in case the elasticsearch index pattern is modified.
-#setup.template.name: "packetbeat-%{[beat.version]}"
+setup.template.name: "packetbeat-%{[beat.version]}"
 
 # Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings.
 # The first part is the version of the beat and then -* is used to match all daily indices.
 # The template name and pattern has to be set in case the elasticsearch index pattern is modified.
-#setup.template.pattern: "packetbeat-%{[beat.version]}-*"
+setup.template.pattern: "packetbeat-%{[beat.version]}-*"
 
 # Path to fields.yml file to generate the template
-#setup.template.fields: "${path.config}/fields.yml"
+setup.template.fields: "${path.config}/fields.yml"
 
 # Overwrite existing template
 setup.template.overwrite: true