From 83a64509c5960fb04d2ae9437040ebeaed1f177f Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 11 Apr 2018 21:24:46 -0500 Subject: [PATCH] Update README, beat deployment, and configs Beat set is using a loop to ship both templates and dashboards using different commands. This is being done to ensure no data is lost or mis-setup during index creation. Packetbeat only needs to be installed on hosts All of the beats are now using standard field templates. Change-Id: Ie6220eb94a12780ad122ba367bf7654d97c212e8 Signed-off-by: Kevin Carter --- elk_metrics_6x/installAuditbeat.yml | 12 +++ elk_metrics_6x/installMetricbeat.yml | 4 + elk_metrics_6x/installPacketbeat.yml | 13 ++- elk_metrics_6x/readme.rst | 25 +++--- elk_metrics_6x/templates/auditbeat.yml.j2 | 27 +++--- elk_metrics_6x/templates/metricbeat.yml.j2 | 22 ++--- elk_metrics_6x/templates/packetbeat.yml.j2 | 99 +++++++++++++++------- 7 files changed, 134 insertions(+), 68 deletions(-) diff --git a/elk_metrics_6x/installAuditbeat.yml b/elk_metrics_6x/installAuditbeat.yml index ac72f963..f167c871 100644 --- a/elk_metrics_6x/installAuditbeat.yml +++ b/elk_metrics_6x/installAuditbeat.yml @@ -26,6 +26,14 @@ src: templates/auditbeat.yml.j2 dest: /etc/auditbeat/auditbeat.yml + - name: Stop auditd + systemd: + name: "auditd" + enabled: "{{ not inventory_hostname in groups['kibana'] | default([]) }}" + state: stopped + when: + - not apply_security_hardening | default(true) | bool + - name: Enable and restart auditbeat systemd: name: "auditbeat" @@ -48,6 +56,10 @@ {% endfor %} {% set elasticsearch_hosts = [IP_ARR | map('regex_replace', '$', ':' ~ elastic_port|string()) | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' )] %} auditbeat setup + {{ item }} -E 'output.logstash.enabled=false' -E 'output.elasticsearch.hosts={{ elasticsearch_hosts }}' -e -v + with_items: + - "--template" + - "--dashboards" diff --git a/elk_metrics_6x/installMetricbeat.yml b/elk_metrics_6x/installMetricbeat.yml index 2f1e8c11..baa48ab3 100644 --- a/elk_metrics_6x/installMetricbeat.yml +++ b/elk_metrics_6x/installMetricbeat.yml @@ -72,6 +72,10 @@ {% endfor %} {% set elasticsearch_hosts = [IP_ARR | map('regex_replace', '$', ':' ~ elastic_port|string()) | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' )] %} metricbeat setup + {{ item }} -E 'output.logstash.enabled=false' -E 'output.elasticsearch.hosts={{ elasticsearch_hosts }}' -e -v + with_items: + - "--template" + - "--dashboards" diff --git a/elk_metrics_6x/installPacketbeat.yml b/elk_metrics_6x/installPacketbeat.yml index d91c2bbe..3d787ec9 100644 --- a/elk_metrics_6x/installPacketbeat.yml +++ b/elk_metrics_6x/installPacketbeat.yml @@ -1,6 +1,6 @@ --- - name: Install Packetbeat - hosts: all + hosts: hosts become: true vars: haproxy_ssl: false @@ -13,9 +13,12 @@ - name: Ensure packetbeat is installed apt: - name: packetbeat + name: "{{ item }}" state: present update_cache: true + with_items: + - tcpdump + - packetbeat post_tasks: - name: Drop packetbeat conf file @@ -31,7 +34,7 @@ - name: Load Packetbeat Dashboards - hosts: all[0] + hosts: hosts[0] become: true vars_files: - vars/variables.yml @@ -45,6 +48,10 @@ {% endfor %} {% set elasticsearch_hosts = [IP_ARR | map('regex_replace', '$', ':' ~ elastic_port|string()) | map('regex_replace', '$', '"') | map('regex_replace', '^', '"') | list | join(',' )] %} packetbeat setup + {{ item }} -E 'output.logstash.enabled=false' -E 'output.elasticsearch.hosts={{ elasticsearch_hosts }}' -e -v + with_items: + - "--template" + - "--dashboards" diff --git a/elk_metrics_6x/readme.rst b/elk_metrics_6x/readme.rst index 5fc097af..75689310 100644 --- a/elk_metrics_6x/readme.rst +++ b/elk_metrics_6x/readme.rst @@ -1,17 +1,12 @@ -install Elk stack with topbeat to gather metrics -################################################# +Install ELK with beats to gather metrics +######################################## :tags: openstack, ansible - -Changelog ---------- -2018-03-06 Per Abildgaard Toft (per@minfejl.dk): Updated to version Elasticsearch,Logstash and Kibana 6.x. Changed Topebeat (deprecated) to metricbeat. Included haproxy endpoint configuration. - - About this repository --------------------- -This set of playbooks will deploy elk cluster (Elasticsearch, Logstash, Kibana) with topbeat to gather metrics from hosts metrics to the ELK cluster. +This set of playbooks will deploy elk cluster (Elasticsearch, Logstash, Kibana) +with topbeat to gather metrics from hosts metrics to the ELK cluster. Process ------- @@ -27,7 +22,7 @@ Copy the env.d file into place .. code-block:: bash - cd openstack-ansible-ops/elk_metrics_6x + cd /opt/openstack-ansible-ops/elk_metrics_6x cp env.d/elk.yml /etc/openstack_deploy/env.d/ Copy the conf.d file into place @@ -36,7 +31,9 @@ Copy the conf.d file into place cp conf.d/elk.yml /etc/openstack_deploy/conf.d/ -In **elk.yml**, list your logging hosts under elastic-logstash_hosts to create the elasticsearch cluster in multiple containers and one logging host under kibana_hosts to create the kibana container +In **elk.yml**, list your logging hosts under elastic-logstash_hosts to create +the elasticsearch cluster in multiple containers and one logging host under +kibana_hosts to create the kibana container .. code-block:: bash @@ -70,7 +67,8 @@ Install Kibana, nginx reverse proxy and metricbeat on the kibana container cd /opt/openstack-ansible-ops/elk_metrics_6x openstack-ansible installKibana.yml -install Metricbeat everywhere to start shipping metrics to our logstash instances +Install Metricbeat everywhere to start shipping metrics to our logstash +instances .. code-block:: bash @@ -79,7 +77,8 @@ install Metricbeat everywhere to start shipping metrics to our logstash instance Optional | conigure haproxy endpoints -Edit the `/etc/openstack_deploy/user_variables.yml` file and add fiel following lines +Edit the `/etc/openstack_deploy/user_variables.yml` file and add fiel following +lines .. code-block:: yaml diff --git a/elk_metrics_6x/templates/auditbeat.yml.j2 b/elk_metrics_6x/templates/auditbeat.yml.j2 index 812f8cba..f1be66f2 100644 --- a/elk_metrics_6x/templates/auditbeat.yml.j2 +++ b/elk_metrics_6x/templates/auditbeat.yml.j2 @@ -32,7 +32,7 @@ auditbeat.modules: # The auditd module collects events from the audit framework in the Linux # kernel. You need to specify audit rules for the events that you want to audit. - module: auditd - socket_type: multicast + socket_type: {{ (apply_security_hardening | default(true) | bool) | ternary('multicast', 'unicast') }} resolve_ids: true failure_mode: silent backlog_limit: 8196 @@ -55,14 +55,18 @@ auditbeat.modules: ## Executions. -a always,exit -F arch=b64 -S execve,execveat -k exec - ## Identity changes. + # Things that affect identity. -w /etc/group -p wa -k identity -w /etc/passwd -p wa -k identity -w /etc/gshadow -p wa -k identity + -w /etc/shadow -p wa -k identity + + # Unauthorized access attempts to files (unsuccessful). + -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -F auid>=1000 -F auid!=4294967295 -F key=access + -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -F auid>=1000 -F auid!=4294967295 -F key=access + -a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EACCES -F auid>=1000 -F auid!=4294967295 -F key=access + -a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EPERM -F auid>=1000 -F auid!=4294967295 -F key=access - ## Unauthorized access attempts. - -a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EACCES -k access - -a always,exit -F arch=b64 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access {% endif %} # The file integrity module sends events when files are changed (created, @@ -71,11 +75,12 @@ auditbeat.modules: paths: - /bin - /etc/apt - - /etc/networking + - /etc/network - /etc/openstack_deploy - /etc/sysconfig - /etc/systemd - /etc/yum + - /etc/zypp - /openstack/venvs - /sbin - /usr/bin @@ -96,11 +101,11 @@ auditbeat.modules: # Average scan rate. This throttles the amount of CPU and I/O that Auditbeat # consumes at startup while scanning. Default is "50 MiB". - scan_rate_per_sec: 50 MiB + scan_rate_per_sec: 64 MiB # Limit on the size of files that will be hashed. Default is "100 MiB". # Limit on the size of files that will be hashed. Default is "100 MiB". - max_file_size: 100 MiB + max_file_size: 128 MiB # Hash types to compute when the file changes. Supported types are # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, @@ -774,15 +779,15 @@ setup.template.enabled: true # Template name. By default the template name is "auditbeat-%{[beat.version]}" # The template name and pattern has to be set in case the elasticsearch index pattern is modified. -#setup.template.name: "auditbeat-%{[beat.version]}" +setup.template.name: "auditbeat-%{[beat.version]}" # Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. # The first part is the version of the beat and then -* is used to match all daily indices. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. -#setup.template.pattern: "auditbeat-%{[beat.version]}-*" +setup.template.pattern: "auditbeat-%{[beat.version]}-*" # Path to fields.yml file to generate the template -#setup.template.fields: "${path.config}/fields.yml" +setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: true diff --git a/elk_metrics_6x/templates/metricbeat.yml.j2 b/elk_metrics_6x/templates/metricbeat.yml.j2 index 907ef4a0..be756008 100644 --- a/elk_metrics_6x/templates/metricbeat.yml.j2 +++ b/elk_metrics_6x/templates/metricbeat.yml.j2 @@ -79,21 +79,21 @@ metricbeat.modules: # in the top N by CPU or memory, in order to reduce the number of documents created. # If both the `by_cpu` and `by_memory` options are used, the union of the two sets # is included. - #process.include_top_n: + process.include_top_n: # # Set to false to disable this feature and include all processes - #enabled: true + enabled: true # How many processes to include from the top by CPU. The processes are sorted # by the `system.process.cpu.total.pct` field. - #by_cpu: 0 + by_cpu: 20 # How many processes to include from the top by memory. The processes are sorted # by the `system.process.memory.rss.bytes` field. - #by_memory: 0 + by_memory: 20 # If false, cmdline of a process is not cached. - #process.cmdline.cache.enabled: true + process.cmdline.cache.enabled: true # Enable collection of cgroup metrics from processes on Linux. process.cgroups.enabled: true @@ -107,9 +107,9 @@ metricbeat.modules: process.include_cpu_ticks: {{ inventory_hostname == physical_host }} # Configure reverse DNS lookup on remote IP addresses in the socket metricset. - #socket.reverse_lookup.enabled: false - #socket.reverse_lookup.success_ttl: 60s - #socket.reverse_lookup.failure_ttl: 60s + socket.reverse_lookup.enabled: true + socket.reverse_lookup.success_ttl: 60s + socket.reverse_lookup.failure_ttl: 60s ##------------------------------ Aerospike Module ----------------------------- #- module: aerospike @@ -1165,15 +1165,15 @@ setup.template.enabled: true # Template name. By default the template name is "metricbeat-%{[beat.version]}" # The template name and pattern has to be set in case the elasticsearch index pattern is modified. -#setup.template.name: "metricbeat-%{[beat.version]}" +setup.template.name: "metricbeat-%{[beat.version]}" # Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. # The first part is the version of the beat and then -* is used to match all daily indices. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. -#setup.template.pattern: "metricbeat-%{[beat.version]}-*" +setup.template.pattern: "metricbeat-%{[beat.version]}-*" # Path to fields.yml file to generate the template -#setup.template.fields: "${path.config}/fields.yml" +setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: true diff --git a/elk_metrics_6x/templates/packetbeat.yml.j2 b/elk_metrics_6x/templates/packetbeat.yml.j2 index 167cc946..0cec98f0 100644 --- a/elk_metrics_6x/templates/packetbeat.yml.j2 +++ b/elk_metrics_6x/templates/packetbeat.yml.j2 @@ -18,24 +18,24 @@ packetbeat.interfaces.device: any # not the fastest option. # * af_packet, which uses memory-mapped sniffing. This option is faster than # libpcap and doesn't require a kernel module, but it's Linux-specific. -#packetbeat.interfaces.type: pcap +packetbeat.interfaces.type: af_packet # The maximum size of the packets to capture. The default is 65535, which is # large enough for almost all networks and interface types. If you sniff on a # physical network interface, the optimal setting is the MTU size. On virtual # interfaces, however, it's safer to accept the default value. -#packetbeat.interfaces.snaplen: 65535 +packetbeat.interfaces.snaplen: 65535 # The maximum size of the shared memory buffer to use between the kernel and # user space. A bigger buffer usually results in lower CPU usage, but consumes # more memory. This setting is only available for the af_packet sniffer type. # The default is 30 MB. -#packetbeat.interfaces.buffer_size_mb: 30 +packetbeat.interfaces.buffer_size_mb: 30 # Packetbeat automatically generates a BPF for capturing only the traffic on # ports where it expects to find known protocols. Use this settings to tell # Packetbeat to generate a BPF filter that accepts VLAN tags. -#packetbeat.interfaces.with_vlans: true +packetbeat.interfaces.with_vlans: true # Use this setting to override the automatically generated BPF filter. #packetbeat.interfaces.bpf_filter: @@ -44,7 +44,7 @@ packetbeat.interfaces.device: any packetbeat.flows: # Enable Network flows. Default: true - #enabled: true + enabled: true # Set network flow timeout. Flow is killed if no packet is received before being # timed out. @@ -60,10 +60,17 @@ packetbeat.protocols: # Enable ICMPv4 and ICMPv6 monitoring. Default: true enabled: true -{% if inventory_hostname in groups['rabbitmq_all'] | default([]) %} - type: amqp # Enable AMQP monitoring. Default: true - enabled: true +{% set ns = namespace(enabled=(inventory_hostname in groups['rabbitmq_all'] | default([]))) %} +{% if not ns.enabled | bool %} +{% for _item in groups['rabbitmq_all'] | default([]) %} +{% if not ns.enabled | bool | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %} +{% set ns.enabled = true %} +{% endif %} +{% endfor %} +{% endif %} + enabled: {{ ns.enabled | bool }} # Configure the ports where to listen for AMQP traffic. You can disable # the AMQP protocol by commenting out the list of ports. @@ -97,9 +104,11 @@ packetbeat.protocols: # Transaction timeout. Expired transactions will no longer be correlated to # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -{% endif %} - type: cassandra + # Enable cassandra monitoring. Default: false + enabled: false + #Cassandra port for traffic monitoring. ports: [9042] @@ -156,10 +165,17 @@ packetbeat.protocols: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -{% if inventory_hostname in groups['shared-infra_hosts'] | default([]) %} - type: http # Enable HTTP monitoring. Default: true - enabled: true +{% set ns = namespace(enabled=(inventory_hostname in groups['shared-infra_hosts'] | default([]))) %} +{% if not ns.enabled | bool %} +{% for _item in groups['shared-infra_hosts'] | default([]) %} +{% if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %} +{% set ns.enabled = true %} +{% endif %} +{% endfor %} +{% endif %} + enabled: {{ ns.enabled }} # Configure the ports where to listen for HTTP traffic. You can disable # the HTTP protocol by commenting out the list of ports. @@ -209,12 +225,18 @@ packetbeat.protocols: # Maximum message size. If an HTTP message is larger than this, it will # be trimmed to this size. Default is 10 MB. #max_message_size: 10485760 -{% endif %} -{% if inventory_hostname in groups['memcached_all'] | default([]) %} - type: memcache # Enable memcache monitoring. Default: true - enabled: true +{% set ns = namespace(enabled=(inventory_hostname in groups['memcached_all'] | default([]))) %} +{% if not ns.enabled | bool %} +{% for _item in groups['memcached_all'] | default([]) %} +{% if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %} +{% set ns.enabled = true %} +{% endif %} +{% endfor %} +{% endif %} + enabled: {{ ns.enabled }} # Configure the ports where to listen for memcache traffic. You can disable # the Memcache protocol by commenting out the list of ports. @@ -260,12 +282,18 @@ packetbeat.protocols: # Transaction timeout. Expired transactions will no longer be correlated to # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -{% endif %} -{% if inventory_hostname in groups['galera_all'] | default([]) %} - type: mysql # Enable mysql monitoring. Default: true - enabled: true +{% set ns = namespace(enabled=(inventory_hostname in groups['galera_all'] | default([]))) %} +{% if not ns.enabled | bool %} +{% for _item in groups['galera_all'] | default([]) %} +{% if not ns.enabled | bool and _item in groups[inventory_hostname + '-host_containers'] | default([]) %} +{% set ns.enabled = true %} +{% endif %} +{% endfor %} +{% endif %} + enabled: {{ ns.enabled }} # Configure the ports where to listen for MySQL traffic. You can disable # the MySQL protocol by commenting out the list of ports. @@ -282,11 +310,10 @@ packetbeat.protocols: # Transaction timeout. Expired transactions will no longer be correlated to # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -{% endif %} - type: pgsql # Enable pgsql monitoring. Default: true - #enabled: true + enabled: false # Configure the ports where to listen for Pgsql traffic. You can disable # the Pgsql protocol by commenting out the list of ports. @@ -306,7 +333,7 @@ packetbeat.protocols: - type: redis # Enable redis monitoring. Default: true - #enabled: true + enabled: false # Configure the ports where to listen for Redis traffic. You can disable # the Redis protocol by commenting out the list of ports. @@ -326,7 +353,7 @@ packetbeat.protocols: - type: thrift # Enable thrift monitoring. Default: true - #enabled: true + enabled: false # Configure the ports where to listen for Thrift-RPC traffic. You can disable # the Thrift-RPC protocol by commenting out the list of ports. @@ -381,7 +408,7 @@ packetbeat.protocols: - type: mongodb # Enable mongodb monitoring. Default: true - #enabled: true + enabled: false # Configure the ports where to listen for MongoDB traffic. You can disable # the MongoDB protocol by commenting out the list of ports. @@ -409,10 +436,17 @@ packetbeat.protocols: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -{% if (inventory_hostname in groups['glance_all'] | default([])) or (inventory_hostname in groups['nova_compute'] | default([])) %} - type: nfs # Enable NFS monitoring. Default: true - enabled: true +{% set ns = namespace(enabled=((inventory_hostname in groups['glance_all'] | default([])) or (inventory_hostname in groups['nova_compute'] | default([])))) %} +{% if not ns.enabled | bool %} +{% for _item in groups['glance_all'] | default([]) + groups['nova_compute'] | default([]) %} +{% if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %} +{% set ns.enabled = true %} +{% endif %} +{% endfor %} +{% endif %} + enabled: {{ ns.enabled }} # Configure the ports where to listen for NFS traffic. You can disable # the NFS protocol by commenting out the list of ports. @@ -429,12 +463,18 @@ packetbeat.protocols: # Transaction timeout. Expired transactions will no longer be correlated to # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -{% endif %} -{% if (inventory_hostname in groups['haproxy_all'] | default([])) or (inventory_hostname in groups['horizon_alll'] | default([])) %} - type: tls # Enable TLS monitoring. Default: true - #enabled: true +{% set ns = namespace(enabled=((inventory_hostname in groups['haproxy_all'] | default([])) or (inventory_hostname in groups['horizon_all'] | default([])))) %} +{% if not ns.enabled | bool %} +{% for _item in groups['haproxy_all'] | default([]) + groups['horizon_all'] | default([]) %} +{% if not ns.enabled | bool or _item in groups[inventory_hostname + '-host_containers'] | default([]) %} +{% set ns.enabled = true %} +{% endif %} +{% endfor %} +{% endif %} + enabled: {{ ns.enabled }} # Configure the ports where to listen for TLS traffic. You can disable # the TLS protocol by commenting out the list of ports. @@ -447,7 +487,6 @@ packetbeat.protocols: # If this option is enabled, the raw certificates will be stored # in PEM format under the `raw` key. The default is false. #include_raw_certificates: false -{% endif %} #=========================== Monitored processes ============================== @@ -1127,15 +1166,15 @@ setup.template.enabled: true # Template name. By default the template name is "packetbeat-%{[beat.version]}" # The template name and pattern has to be set in case the elasticsearch index pattern is modified. -#setup.template.name: "packetbeat-%{[beat.version]}" +setup.template.name: "packetbeat-%{[beat.version]}" # Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. # The first part is the version of the beat and then -* is used to match all daily indices. # The template name and pattern has to be set in case the elasticsearch index pattern is modified. -#setup.template.pattern: "packetbeat-%{[beat.version]}-*" +setup.template.pattern: "packetbeat-%{[beat.version]}-*" # Path to fields.yml file to generate the template -#setup.template.fields: "${path.config}/fields.yml" +setup.template.fields: "${path.config}/fields.yml" # Overwrite existing template setup.template.overwrite: true