Update README, beat deployment, and configs

Beat set is using a loop to ship both templates and dashboards using
different commands. This is being done to ensure no data is lost or
mis-setup during index creation.

Packetbeat only needs to be installed on hosts

All of the beats are now using standard field templates.

Change-Id: Ie6220eb94a12780ad122ba367bf7654d97c212e8
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
Kevin Carter 2018-04-11 21:24:46 -05:00
parent 17fb37f075
commit 83a64509c5
No known key found for this signature in database
GPG Key ID: 9443251A787B9FB3
7 changed files with 134 additions and 68 deletions

View File

@ -26,6 +26,14 @@
src: templates/auditbeat.yml.j2
dest: /etc/auditbeat/auditbeat.yml
- name: Stop auditd
systemd:
name: "auditd"
enabled: "{{ not inventory_hostname in groups['kibana'] | default([]) }}"
state: stopped
when:
- not apply_security_hardening | default(true) | bool
- name: Enable and restart auditbeat
systemd:
name: "auditbeat"
@ -48,6 +56,10 @@
{% endfor %}
{% set elasticsearch_hosts = [IP_ARR | map('regex_replace', '$', ':' ~ elastic_port|string()) | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' )] %}
auditbeat setup
{{ item }}
-E 'output.logstash.enabled=false'
-E 'output.elasticsearch.hosts={{ elasticsearch_hosts }}'
-e -v
with_items:
- "--template"
- "--dashboards"

View File

@ -72,6 +72,10 @@
{% endfor %}
{% set elasticsearch_hosts = [IP_ARR | map('regex_replace', '$', ':' ~ elastic_port|string()) | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' )] %}
metricbeat setup
{{ item }}
-E 'output.logstash.enabled=false'
-E 'output.elasticsearch.hosts={{ elasticsearch_hosts }}'
-e -v
with_items:
- "--template"
- "--dashboards"

View File

@ -1,6 +1,6 @@
---
- name: Install Packetbeat
hosts: all
hosts: hosts
become: true
vars:
haproxy_ssl: false
@ -13,9 +13,12 @@
- name: Ensure packetbeat is installed
apt:
name: packetbeat
name: "{{ item }}"
state: present
update_cache: true
with_items:
- tcpdump
- packetbeat
post_tasks:
- name: Drop packetbeat conf file
@ -31,7 +34,7 @@
- name: Load Packetbeat Dashboards
hosts: all[0]
hosts: hosts[0]
become: true
vars_files:
- vars/variables.yml
@ -45,6 +48,10 @@
{% endfor %}
{% set elasticsearch_hosts = [IP_ARR | map('regex_replace', '$', ':' ~ elastic_port|string()) | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' )] %}
packetbeat setup
{{ item }}
-E 'output.logstash.enabled=false'
-E 'output.elasticsearch.hosts={{ elasticsearch_hosts }}'
-e -v
with_items:
- "--template"
- "--dashboards"

View File

@ -1,17 +1,12 @@
install Elk stack with topbeat to gather metrics
#################################################
Install ELK with beats to gather metrics
########################################
:tags: openstack, ansible
Changelog
---------
2018-03-06 Per Abildgaard Toft (per@minfejl.dk): Updated to version Elasticsearch,Logstash and Kibana 6.x. Changed Topebeat (deprecated) to metricbeat. Included haproxy endpoint configuration.
About this repository
---------------------
This set of playbooks will deploy elk cluster (Elasticsearch, Logstash, Kibana) with topbeat to gather metrics from hosts metrics to the ELK cluster.
This set of playbooks will deploy elk cluster (Elasticsearch, Logstash, Kibana)
with topbeat to gather metrics from hosts metrics to the ELK cluster.
Process
-------
@ -27,7 +22,7 @@ Copy the env.d file into place
.. code-block:: bash
cd openstack-ansible-ops/elk_metrics_6x
cd /opt/openstack-ansible-ops/elk_metrics_6x
cp env.d/elk.yml /etc/openstack_deploy/env.d/
Copy the conf.d file into place
@ -36,7 +31,9 @@ Copy the conf.d file into place
cp conf.d/elk.yml /etc/openstack_deploy/conf.d/
In **elk.yml**, list your logging hosts under elastic-logstash_hosts to create the elasticsearch cluster in multiple containers and one logging host under kibana_hosts to create the kibana container
In **elk.yml**, list your logging hosts under elastic-logstash_hosts to create
the elasticsearch cluster in multiple containers and one logging host under
kibana_hosts to create the kibana container
.. code-block:: bash
@ -70,7 +67,8 @@ Install Kibana, nginx reverse proxy and metricbeat on the kibana container
cd /opt/openstack-ansible-ops/elk_metrics_6x
openstack-ansible installKibana.yml
install Metricbeat everywhere to start shipping metrics to our logstash instances
Install Metricbeat everywhere to start shipping metrics to our logstash
instances
.. code-block:: bash
@ -79,7 +77,8 @@ install Metricbeat everywhere to start shipping metrics to our logstash instance
Optional | conigure haproxy endpoints
Edit the `/etc/openstack_deploy/user_variables.yml` file and add fiel following lines
Edit the `/etc/openstack_deploy/user_variables.yml` file and add fiel following
lines
.. code-block:: yaml

View File

@ -32,7 +32,7 @@ auditbeat.modules:
# The auditd module collects events from the audit framework in the Linux
# kernel. You need to specify audit rules for the events that you want to audit.
- module: auditd
socket_type: multicast
socket_type: {{ (apply_security_hardening | default(true) | bool) | ternary('multicast', 'unicast') }}
resolve_ids: true
failure_mode: silent
backlog_limit: 8196
@ -55,14 +55,18 @@ auditbeat.modules:
## Executions.
-a always,exit -F arch=b64 -S execve,execveat -k exec
## Identity changes.
# Things that affect identity.
-w /etc/group -p wa -k identity
-w /etc/passwd -p wa -k identity
-w /etc/gshadow -p wa -k identity
-w /etc/shadow -p wa -k identity
# Unauthorized access attempts to files (unsuccessful).
-a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -F auid>=1000 -F auid!=4294967295 -F key=access
-a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -F auid>=1000 -F auid!=4294967295 -F key=access
-a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EACCES -F auid>=1000 -F auid!=4294967295 -F key=access
-a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EPERM -F auid>=1000 -F auid!=4294967295 -F key=access
## Unauthorized access attempts.
-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access
-a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access
{% endif %}
# The file integrity module sends events when files are changed (created,
@ -71,11 +75,12 @@ auditbeat.modules:
paths:
- /bin
- /etc/apt
- /etc/networking
- /etc/network
- /etc/openstack_deploy
- /etc/sysconfig
- /etc/systemd
- /etc/yum
- /etc/zypp
- /openstack/venvs
- /sbin
- /usr/bin
@ -96,11 +101,11 @@ auditbeat.modules:
# Average scan rate. This throttles the amount of CPU and I/O that Auditbeat
# consumes at startup while scanning. Default is "50 MiB".
scan_rate_per_sec: 50 MiB
scan_rate_per_sec: 64 MiB
# Limit on the size of files that will be hashed. Default is "100 MiB".
# Limit on the size of files that will be hashed. Default is "100 MiB".
max_file_size: 100 MiB
max_file_size: 128 MiB
# Hash types to compute when the file changes. Supported types are
# blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384,
@ -774,15 +779,15 @@ setup.template.enabled: true
# Template name. By default the template name is "auditbeat-%{[beat.version]}"
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.name: "auditbeat-%{[beat.version]}"
setup.template.name: "auditbeat-%{[beat.version]}"
# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings.
# The first part is the version of the beat and then -* is used to match all daily indices.
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.pattern: "auditbeat-%{[beat.version]}-*"
setup.template.pattern: "auditbeat-%{[beat.version]}-*"
# Path to fields.yml file to generate the template
#setup.template.fields: "${path.config}/fields.yml"
setup.template.fields: "${path.config}/fields.yml"
# Overwrite existing template
setup.template.overwrite: true

View File

@ -79,21 +79,21 @@ metricbeat.modules:
# in the top N by CPU or memory, in order to reduce the number of documents created.
# If both the `by_cpu` and `by_memory` options are used, the union of the two sets
# is included.
#process.include_top_n:
process.include_top_n:
#
# Set to false to disable this feature and include all processes
#enabled: true
enabled: true
# How many processes to include from the top by CPU. The processes are sorted
# by the `system.process.cpu.total.pct` field.
#by_cpu: 0
by_cpu: 20
# How many processes to include from the top by memory. The processes are sorted
# by the `system.process.memory.rss.bytes` field.
#by_memory: 0
by_memory: 20
# If false, cmdline of a process is not cached.
#process.cmdline.cache.enabled: true
process.cmdline.cache.enabled: true
# Enable collection of cgroup metrics from processes on Linux.
process.cgroups.enabled: true
@ -107,9 +107,9 @@ metricbeat.modules:
process.include_cpu_ticks: {{ inventory_hostname == physical_host }}
# Configure reverse DNS lookup on remote IP addresses in the socket metricset.
#socket.reverse_lookup.enabled: false
#socket.reverse_lookup.success_ttl: 60s
#socket.reverse_lookup.failure_ttl: 60s
socket.reverse_lookup.enabled: true
socket.reverse_lookup.success_ttl: 60s
socket.reverse_lookup.failure_ttl: 60s
##------------------------------ Aerospike Module -----------------------------
#- module: aerospike
@ -1165,15 +1165,15 @@ setup.template.enabled: true
# Template name. By default the template name is "metricbeat-%{[beat.version]}"
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.name: "metricbeat-%{[beat.version]}"
setup.template.name: "metricbeat-%{[beat.version]}"
# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings.
# The first part is the version of the beat and then -* is used to match all daily indices.
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.pattern: "metricbeat-%{[beat.version]}-*"
setup.template.pattern: "metricbeat-%{[beat.version]}-*"
# Path to fields.yml file to generate the template
#setup.template.fields: "${path.config}/fields.yml"
setup.template.fields: "${path.config}/fields.yml"
# Overwrite existing template
setup.template.overwrite: true

View File

@ -18,24 +18,24 @@ packetbeat.interfaces.device: any
# not the fastest option.
# * af_packet, which uses memory-mapped sniffing. This option is faster than
# libpcap and doesn't require a kernel module, but it's Linux-specific.
#packetbeat.interfaces.type: pcap
packetbeat.interfaces.type: af_packet
# The maximum size of the packets to capture. The default is 65535, which is
# large enough for almost all networks and interface types. If you sniff on a
# physical network interface, the optimal setting is the MTU size. On virtual
# interfaces, however, it's safer to accept the default value.
#packetbeat.interfaces.snaplen: 65535
packetbeat.interfaces.snaplen: 65535
# The maximum size of the shared memory buffer to use between the kernel and
# user space. A bigger buffer usually results in lower CPU usage, but consumes
# more memory. This setting is only available for the af_packet sniffer type.
# The default is 30 MB.
#packetbeat.interfaces.buffer_size_mb: 30
packetbeat.interfaces.buffer_size_mb: 30
# Packetbeat automatically generates a BPF for capturing only the traffic on
# ports where it expects to find known protocols. Use this settings to tell
# Packetbeat to generate a BPF filter that accepts VLAN tags.
#packetbeat.interfaces.with_vlans: true
packetbeat.interfaces.with_vlans: true
# Use this setting to override the automatically generated BPF filter.
#packetbeat.interfaces.bpf_filter:
@ -44,7 +44,7 @@ packetbeat.interfaces.device: any
packetbeat.flows:
# Enable Network flows. Default: true
#enabled: true
enabled: true
# Set network flow timeout. Flow is killed if no packet is received before being
# timed out.
@ -60,10 +60,17 @@ packetbeat.protocols:
# Enable ICMPv4 and ICMPv6 monitoring. Default: true
enabled: true
{% if inventory_hostname in groups['rabbitmq_all'] | default([]) %}
- type: amqp
# Enable AMQP monitoring. Default: true
enabled: true
{% set ns = namespace(enabled=(inventory_hostname in groups['rabbitmq_all'] | default([]))) %}
{% if not ns.enabled | bool %}
{% for _item in groups['rabbitmq_all'] | default([]) %}
{% if not ns.enabled | bool | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
{% set ns.enabled = true %}
{% endif %}
{% endfor %}
{% endif %}
enabled: {{ ns.enabled | bool }}
# Configure the ports where to listen for AMQP traffic. You can disable
# the AMQP protocol by commenting out the list of ports.
@ -97,9 +104,11 @@ packetbeat.protocols:
# Transaction timeout. Expired transactions will no longer be correlated to
# incoming responses, but sent to Elasticsearch immediately.
#transaction_timeout: 10s
{% endif %}
- type: cassandra
# Enable cassandra monitoring. Default: false
enabled: false
#Cassandra port for traffic monitoring.
ports: [9042]
@ -156,10 +165,17 @@ packetbeat.protocols:
# incoming responses, but sent to Elasticsearch immediately.
#transaction_timeout: 10s
{% if inventory_hostname in groups['shared-infra_hosts'] | default([]) %}
- type: http
# Enable HTTP monitoring. Default: true
enabled: true
{% set ns = namespace(enabled=(inventory_hostname in groups['shared-infra_hosts'] | default([]))) %}
{% if not ns.enabled | bool %}
{% for _item in groups['shared-infra_hosts'] | default([]) %}
{% if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
{% set ns.enabled = true %}
{% endif %}
{% endfor %}
{% endif %}
enabled: {{ ns.enabled }}
# Configure the ports where to listen for HTTP traffic. You can disable
# the HTTP protocol by commenting out the list of ports.
@ -209,12 +225,18 @@ packetbeat.protocols:
# Maximum message size. If an HTTP message is larger than this, it will
# be trimmed to this size. Default is 10 MB.
#max_message_size: 10485760
{% endif %}
{% if inventory_hostname in groups['memcached_all'] | default([]) %}
- type: memcache
# Enable memcache monitoring. Default: true
enabled: true
{% set ns = namespace(enabled=(inventory_hostname in groups['memcached_all'] | default([]))) %}
{% if not ns.enabled | bool %}
{% for _item in groups['memcached_all'] | default([]) %}
{% if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
{% set ns.enabled = true %}
{% endif %}
{% endfor %}
{% endif %}
enabled: {{ ns.enabled }}
# Configure the ports where to listen for memcache traffic. You can disable
# the Memcache protocol by commenting out the list of ports.
@ -260,12 +282,18 @@ packetbeat.protocols:
# Transaction timeout. Expired transactions will no longer be correlated to
# incoming responses, but sent to Elasticsearch immediately.
#transaction_timeout: 10s
{% endif %}
{% if inventory_hostname in groups['galera_all'] | default([]) %}
- type: mysql
# Enable mysql monitoring. Default: true
enabled: true
{% set ns = namespace(enabled=(inventory_hostname in groups['galera_all'] | default([]))) %}
{% if not ns.enabled | bool %}
{% for _item in groups['galera_all'] | default([]) %}
{% if not ns.enabled | bool and _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
{% set ns.enabled = true %}
{% endif %}
{% endfor %}
{% endif %}
enabled: {{ ns.enabled }}
# Configure the ports where to listen for MySQL traffic. You can disable
# the MySQL protocol by commenting out the list of ports.
@ -282,11 +310,10 @@ packetbeat.protocols:
# Transaction timeout. Expired transactions will no longer be correlated to
# incoming responses, but sent to Elasticsearch immediately.
#transaction_timeout: 10s
{% endif %}
- type: pgsql
# Enable pgsql monitoring. Default: true
#enabled: true
enabled: false
# Configure the ports where to listen for Pgsql traffic. You can disable
# the Pgsql protocol by commenting out the list of ports.
@ -306,7 +333,7 @@ packetbeat.protocols:
- type: redis
# Enable redis monitoring. Default: true
#enabled: true
enabled: false
# Configure the ports where to listen for Redis traffic. You can disable
# the Redis protocol by commenting out the list of ports.
@ -326,7 +353,7 @@ packetbeat.protocols:
- type: thrift
# Enable thrift monitoring. Default: true
#enabled: true
enabled: false
# Configure the ports where to listen for Thrift-RPC traffic. You can disable
# the Thrift-RPC protocol by commenting out the list of ports.
@ -381,7 +408,7 @@ packetbeat.protocols:
- type: mongodb
# Enable mongodb monitoring. Default: true
#enabled: true
enabled: false
# Configure the ports where to listen for MongoDB traffic. You can disable
# the MongoDB protocol by commenting out the list of ports.
@ -409,10 +436,17 @@ packetbeat.protocols:
# incoming responses, but sent to Elasticsearch immediately.
#transaction_timeout: 10s
{% if (inventory_hostname in groups['glance_all'] | default([])) or (inventory_hostname in groups['nova_compute'] | default([])) %}
- type: nfs
# Enable NFS monitoring. Default: true
enabled: true
{% set ns = namespace(enabled=((inventory_hostname in groups['glance_all'] | default([])) or (inventory_hostname in groups['nova_compute'] | default([])))) %}
{% if not ns.enabled | bool %}
{% for _item in groups['glance_all'] | default([]) + groups['nova_compute'] | default([]) %}
{% if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
{% set ns.enabled = true %}
{% endif %}
{% endfor %}
{% endif %}
enabled: {{ ns.enabled }}
# Configure the ports where to listen for NFS traffic. You can disable
# the NFS protocol by commenting out the list of ports.
@ -429,12 +463,18 @@ packetbeat.protocols:
# Transaction timeout. Expired transactions will no longer be correlated to
# incoming responses, but sent to Elasticsearch immediately.
#transaction_timeout: 10s
{% endif %}
{% if (inventory_hostname in groups['haproxy_all'] | default([])) or (inventory_hostname in groups['horizon_alll'] | default([])) %}
- type: tls
# Enable TLS monitoring. Default: true
#enabled: true
{% set ns = namespace(enabled=((inventory_hostname in groups['haproxy_all'] | default([])) or (inventory_hostname in groups['horizon_all'] | default([])))) %}
{% if not ns.enabled | bool %}
{% for _item in groups['haproxy_all'] | default([]) + groups['horizon_all'] | default([]) %}
{% if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %}
{% set ns.enabled = true %}
{% endif %}
{% endfor %}
{% endif %}
enabled: {{ ns.enabled }}
# Configure the ports where to listen for TLS traffic. You can disable
# the TLS protocol by commenting out the list of ports.
@ -447,7 +487,6 @@ packetbeat.protocols:
# If this option is enabled, the raw certificates will be stored
# in PEM format under the `raw` key. The default is false.
#include_raw_certificates: false
{% endif %}
#=========================== Monitored processes ==============================
@ -1127,15 +1166,15 @@ setup.template.enabled: true
# Template name. By default the template name is "packetbeat-%{[beat.version]}"
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.name: "packetbeat-%{[beat.version]}"
setup.template.name: "packetbeat-%{[beat.version]}"
# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings.
# The first part is the version of the beat and then -* is used to match all daily indices.
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.pattern: "packetbeat-%{[beat.version]}-*"
setup.template.pattern: "packetbeat-%{[beat.version]}-*"
# Path to fields.yml file to generate the template
#setup.template.fields: "${path.config}/fields.yml"
setup.template.fields: "${path.config}/fields.yml"
# Overwrite existing template
setup.template.overwrite: true