diff --git a/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml index 0a6dae1a..17d286e0 100644 --- a/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_filebeat/defaults/main.yml @@ -282,3 +282,14 @@ filebeat_prospectors: - /var/log/elasticsearch/*.log tags: - beats + +filebeat_ilm_policy_name: "filebeat-{{ beat_version }}" +filebeat_ilm_policy_template: "filebeat-{{ beat_version }}" +filebeat_ilm_policy_filename: "filebeat-ilm-policy.json" +filebeat_ilm_policy_file_location: "/etc/filebeat" + +ilm_policy_name: "{{ filebeat_ilm_policy_name | default('') }}" +ilm_policy: "{{ (filebeat_ilm_policy | default(default_ilm_policy)) | default({}) }}" +ilm_policy_template: "{{ filebeat_ilm_policy_template | default('') }}" +ilm_policy_filename: "{{ (filebeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}" +ilm_policy_file_location: "{{ (filebeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}" diff --git a/elk_metrics_7x/roles/elastic_filebeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_filebeat/tasks/main.yml index 5fd827e3..68af6604 100644 --- a/elk_metrics_7x/roles/elastic_filebeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_filebeat/tasks/main.yml @@ -82,6 +82,10 @@ notify: - Enable and restart filebeat +- include_role: + name: elastic_ilm + when: ilm_policy.keys() | length > 0 + - name: Run the beat setup role include_role: name: elastic_beat_setup diff --git a/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 b/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 index 85e29526..d3413f7c 100644 --- a/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_filebeat/templates/filebeat.yml.j2 @@ -1368,7 +1368,12 @@ processors: # Configure rollover index pattern. #setup.ilm.pattern: "{now/d}-000001" - +{% if ilm_policy_name is defined %} +setup.ilm.policy_name: "{{ ilm_policy_name }}" +{% endif %} +{% if ilm_policy_file_location is defined %} +setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}" +{% endif %} #============================== Kibana ====================================== {% if (groups['kibana'] | length) > 0 %} diff --git a/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml index d6bbf645..1e4b44c8 100644 --- a/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_heartbeat/defaults/main.yml @@ -14,3 +14,14 @@ # limitations under the License. heartbeat_service_state: restarted + +heartbeat_ilm_policy_name: "heartbeat-{{ beat_version }}" +heartbeat_ilm_policy_template: "heartbeat-{{ beat_version}}" +heartbeat_ilm_policy_filename: "heartbeat-ilm-policy.json" +heartbeat_ilm_policy_file_location: "/etc/heartbeat" + +ilm_policy_name: "{{ heartbeat_ilm_policy_name | default('') }}" +ilm_policy: "{{ (heartbeat_ilm_policy | default(default_ilm_policy)) | default({}) }}" +ilm_policy_template: "{{ heartbeat_ilm_policy_template | default('') }}" +ilm_policy_filename: "{{ (heartbeat_ilm_policy_filename | default(heartbeat_ilm_policy_filename)) | default('') }}" +ilm_policy_file_location: "{{ (heartbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}" diff --git a/elk_metrics_7x/roles/elastic_heartbeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_heartbeat/tasks/main.yml index f6f7ddf8..db454447 100644 --- a/elk_metrics_7x/roles/elastic_heartbeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_heartbeat/tasks/main.yml @@ -88,6 +88,10 @@ notify: - Enable and restart heartbeat +- include_role: + name: elastic_ilm + when: ilm_policy.keys() | length > 0 + - name: Run the beat setup role include_role: name: elastic_beat_setup diff --git a/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 b/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 index d18c5b8c..fa574cc1 100644 --- a/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 @@ -1013,7 +1013,12 @@ processors: # Configure rollover index pattern. #setup.ilm.pattern: "{now/d}-000001" - +{% if ilm_policy_name is defined %} +setup.ilm.policy_name: "{{ ilm_policy_name }}" +{% endif %} +{% if ilm_policy_file_location is defined %} +setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}" +{% endif %} #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} {{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} diff --git a/elk_metrics_7x/roles/elastic_ilm/defaults/main.yml b/elk_metrics_7x/roles/elastic_ilm/defaults/main.yml new file mode 100644 index 00000000..46cde581 --- /dev/null +++ b/elk_metrics_7x/roles/elastic_ilm/defaults/main.yml @@ -0,0 +1,18 @@ +beat_version: "7.3.2" +default_ilm_rollover_max_size: "30G" +default_ilm_rollover_max_age: "15d" +default_ilm_delete_min_age: "30d" +default_ilm_policy: + policy: + phases: + hot: + actions: + rollover: + max_size: "{{ default_ilm_rollover_max_size }}" + max_age: "{{ default_ilm_rollover_max_age }}" + delete: + min_age: "{{ default_ilm_delete_min_age }}" + actions: + delete: {} +default_ilm_policy_filename: "default-ilm-policy.json" +default_ilm_policy_file_location: "/tmp" diff --git a/elk_metrics_7x/roles/elastic_ilm/meta/main.yml b/elk_metrics_7x/roles/elastic_ilm/meta/main.yml new file mode 100644 index 00000000..57592632 --- /dev/null +++ b/elk_metrics_7x/roles/elastic_ilm/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v7.x elastic_ilm role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - trusty + - xenial + - bionic + categories: + - cloud + - development + - elasticsearch + - elastic-stack diff --git a/elk_metrics_7x/roles/elastic_ilm/tasks/elastic_ilm_policy_template.yml b/elk_metrics_7x/roles/elastic_ilm/tasks/elastic_ilm_policy_template.yml new file mode 100644 index 00000000..764324db --- /dev/null +++ b/elk_metrics_7x/roles/elastic_ilm/tasks/elastic_ilm_policy_template.yml @@ -0,0 +1,8 @@ +--- +- name: Template out policy to hosts for initial set up situations + template: + src: "ilm-policy.json.j2" + dest: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}" + mode: 0750 + owner: root + group: root diff --git a/elk_metrics_7x/roles/elastic_ilm/tasks/elastic_ilm_update_policy.yml b/elk_metrics_7x/roles/elastic_ilm/tasks/elastic_ilm_update_policy.yml new file mode 100644 index 00000000..858c4f15 --- /dev/null +++ b/elk_metrics_7x/roles/elastic_ilm/tasks/elastic_ilm_update_policy.yml @@ -0,0 +1,43 @@ +--- +- name: Check if policy exists + uri: + url: "http://127.0.0.1:{{ elastic_port }}/_ilm/policy/{{ ilm_policy_name }}" + method: GET + status_code: 200,404 + body_format: json + register: check_policy + when: ilm_policy_name is defined and ilm_policy is defined + +- name: Update ILM policy when not installing beat for first time + uri: + url: "http://127.0.0.1:{{ elastic_port }}/_ilm/policy/{{ ilm_policy_name }}" + method: PUT + body: "{{ ilm_policy }}" + status_code: 200 + body_format: json + when: check_policy.status == 200 and ilm_policy_name is defined and ilm_policy is defined and (elk_package_state | default('present')) != "latest" + +- name: Get template + uri: + url: "http://127.0.0.1:{{ elastic_port }}/_template/{{ ilm_policy_template }}/" + method: GET + status_code: 200,404 + body_format: json + register: template + when: ilm_policy_template is defined and ilm_policy_name is defined + +- set_fact: + updated_template: "{{ template.json | combine({template.json.keys()[0]|string:{'settings': {'index': {'lifecycle': {'name': 'ilm_policy_name' }}}}}, recursive=True) }}" + when: template.status == 200 and ilm_policy_template is defined and ilm_policy_name is defined + +- name: Update template + uri: + url: "http://127.0.0.1:{{ elastic_port }}/_template/{{ ilm_policy_template }}/" + body: "{{ updated_template[template.json.keys()[0]|string] }}" + method: PUT + body_format: json + status_code: 200 + headers: + Content-Type: "application/json" + kbn-xsrf: "{{ inventory_hostname | to_uuid }}" + when: template.status == 200 and ilm_policy_template is defined and ilm_policy_name is defined diff --git a/elk_metrics_7x/roles/elastic_ilm/tasks/main.yml b/elk_metrics_7x/roles/elastic_ilm/tasks/main.yml new file mode 100644 index 00000000..0bdc3c42 --- /dev/null +++ b/elk_metrics_7x/roles/elastic_ilm/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- include_tasks: "elastic_ilm_policy_template.yml" + when: ilm_policy_file_location is defined and ilm_policy_filename is defined and ilm_policy is defined + +- block: + - name: Update policy and template when not upgrading + include_tasks: "elastic_ilm_update_policy.yml" + delegate_to: "{{ groups['elastic-logstash'][0] }}" + environment: + no_proxy: "{{ hostvars[groups['elastic-logstash'][0]]['ansible_host'] }},127.0.0.1" + run_once: True + when: (elk_package_state | default('present')) != "latest" diff --git a/elk_metrics_7x/roles/elastic_ilm/templates/ilm-policy.json.j2 b/elk_metrics_7x/roles/elastic_ilm/templates/ilm-policy.json.j2 new file mode 100644 index 00000000..f66436d1 --- /dev/null +++ b/elk_metrics_7x/roles/elastic_ilm/templates/ilm-policy.json.j2 @@ -0,0 +1 @@ +{{ ilm_policy | to_json }} diff --git a/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml index 6dbf010e..3f17b7ee 100644 --- a/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_journalbeat/defaults/main.yml @@ -14,3 +14,14 @@ # limitations under the License. journalbeat_service_state: restarted + +journalbeat_ilm_policy_name: "journalbeat-{{ beat_version }}" +journalbeat_ilm_policy_template: "journalbeat-{{ beat_version }}" +journalbeat_ilm_policy_filename: "journalbeat-ilm-policy.json" +journalbeat_ilm_policy_file_location: "/etc/journalbeat" + +ilm_policy_name: "{{ journalbeat_ilm_policy_name | default('') }}" +ilm_policy: "{{ (journalbeat_ilm_policy | default(default_ilm_policy)) | default({}) }}" +ilm_policy_template: "{{ journalbeat_ilm_policy_template | default('') }}" +ilm_policy_filename: "{{ (journalbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}" +ilm_policy_file_location: "{{ (journalbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}" diff --git a/elk_metrics_7x/roles/elastic_journalbeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_journalbeat/tasks/main.yml index e6fee44f..ec3e3d71 100644 --- a/elk_metrics_7x/roles/elastic_journalbeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_journalbeat/tasks/main.yml @@ -88,6 +88,10 @@ notify: - Enable and restart journalbeat +- include_role: + name: elastic_ilm + when: ilm_policy.keys() | length > 0 + - name: Run the beat setup role include_role: name: elastic_beat_setup diff --git a/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 b/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 index 20013905..1d1807f5 100644 --- a/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 @@ -768,7 +768,12 @@ processors: # Configure rollover index pattern. #setup.ilm.pattern: "{now/d}-000001" - +{% if ilm_policy_name is defined %} +setup.ilm.policy_name: "{{ ilm_policy_name }}" +{% endif %} +{% if ilm_policy_file_location is defined %} +setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}" +{% endif %} #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} diff --git a/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml index 1d5b039d..a2ae4065 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_metricbeat/defaults/main.yml @@ -22,3 +22,14 @@ metricbeat_service_state: restarted # Inventory group to configure metricbeat ceph monitoring # via either ceph-restapi or mgr prometheus module ceph_metricbeat_group: cinder_volume + +metricbeat_ilm_policy_name: "metricbeat-{{ beat_version }}" +metricbeat_ilm_policy_template: "metricbeat-{{ beat_version }}" +metricbeat_ilm_policy_filename: "metricbeat-ilm-policy.json" +metricbeat_ilm_policy_file_location: "/etc/metricbeat" + +ilm_policy_name: "{{ metricbeat_ilm_policy_name | default('') }}" +ilm_policy: "{{ (metricbeat_ilm_policy | default(default_ilm_policy)) | default({}) }}" +ilm_policy_template: "{{ metricbeat_ilm_policy_template | default('') }}" +ilm_policy_filename: "{{ (metricbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}" +ilm_policy_file_location: "{{ (metricbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}" diff --git a/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml index 6ca09fb6..3481ec75 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_metricbeat/tasks/main.yml @@ -305,6 +305,10 @@ notify: - Enable and restart metricbeat +- include_role: + name: elastic_ilm + when: ilm_policy.keys() | length > 0 + - name: Run the beat setup role include_role: name: elastic_beat_setup diff --git a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 index f337cb70..e9ab2bd4 100644 --- a/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 @@ -1479,7 +1479,12 @@ processors: # Configure rollover index pattern. #setup.ilm.pattern: "{now/d}-000001" - +{% if ilm_policy_name is defined %} +setup.ilm.policy_name: "{{ ilm_policy_name }}" +{% endif %} +{% if ilm_policy_file_location is defined %} +setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}" +{% endif %} #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} diff --git a/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml b/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml index 85f01f0d..6458501a 100644 --- a/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml +++ b/elk_metrics_7x/roles/elastic_packetbeat/defaults/main.yml @@ -14,3 +14,14 @@ # limitations under the License. packetbeat_service_state: restarted + +packetbeat_ilm_policy_name: "packetbeat-{{ beat_version }}" +packetbeat_ilm_policy_template: "packetbeat-{{ beat_version }}" +packetbeat_ilm_policy_filename: "packetbeat-ilm-policy.json" +packetbeat_ilm_policy_file_location: "/etc/packetbeat" + +ilm_policy_name: "{{ packetbeat_ilm_policy_name | default('') }}" +ilm_policy: "{{ (packetbeat_ilm_policy | default(default_ilm_policy)) | default({}) }}" +ilm_policy_template: "{{ packetbeat_ilm_policy_template | default('') }}" +ilm_policy_filename: "{{ (packetbeat_ilm_policy_filename | default(default_ilm_policy_filename)) | default('') }}" +ilm_policy_file_location: "{{ (packetbeat_ilm_policy_file_location | default(default_ilm_policy_file_location)) | default('') }}" diff --git a/elk_metrics_7x/roles/elastic_packetbeat/tasks/main.yml b/elk_metrics_7x/roles/elastic_packetbeat/tasks/main.yml index 2b5337a0..e1dee70c 100644 --- a/elk_metrics_7x/roles/elastic_packetbeat/tasks/main.yml +++ b/elk_metrics_7x/roles/elastic_packetbeat/tasks/main.yml @@ -73,6 +73,10 @@ notify: - Enable and restart packetbeat +- include_role: + name: elastic_ilm + when: ilm_policy.keys() | length > 0 + - name: Run the beat setup role include_role: name: elastic_beat_setup diff --git a/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 b/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 index 43fb9b8f..dc947f4c 100644 --- a/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 +++ b/elk_metrics_7x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 @@ -1204,7 +1204,12 @@ processors: # Configure rollover index pattern. #setup.ilm.pattern: "{now/d}-000001" - +{% if ilm_policy_name is defined %} +setup.ilm.policy_name: "{{ ilm_policy_name }}" +{% endif %} +{% if ilm_policy_file_location is defined %} +setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}" +{% endif %} #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 %} diff --git a/elk_metrics_7x/vars/variables.yml b/elk_metrics_7x/vars/variables.yml index 48abc1cd..217a85e5 100644 --- a/elk_metrics_7x/vars/variables.yml +++ b/elk_metrics_7x/vars/variables.yml @@ -411,3 +411,22 @@ elastic_beats: hosts: "{{ (groups['nova_compute'] | default([])) | union((groups['utility_all'] | default([]))) | union((groups['memcached_all'] | default([]))) }}" skydive: hosts: "{{ (((groups['skydive_analyzers'] | default([])) | length) > 0) | ternary((groups['hosts'] | default([])), []) }}" + +beat_version: "7.3.2" +default_ilm_rollover_max_size: "30G" +default_ilm_rollover_max_age: "15d" +default_ilm_delete_min_age: "30d" +default_ilm_policy: + policy: + phases: + hot: + actions: + rollover: + max_size: "{{ default_ilm_rollover_max_size }}" + max_age: "{{ default_ilm_rollover_max_age }}" + delete: + min_age: "{{ default_ilm_delete_min_age }}" + actions: + delete: {} +default_ilm_policy_filename: "default-ilm-policy.json" +default_ilm_policy_file_location: "/tmp"