From 3a0b3d2cdeb5bb78ce30002f24e4bca05ab9b40a Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 8 Aug 2018 01:11:51 -0500 Subject: [PATCH] Convert playbooks into roles This change adds the scaffolding required to get multi-distro support running in the roles. The change breaks up our playbooks converting all of the tasks into various roles with internal dependencies. While this will improve execution time, the change is being done to reduce boiler plate and to allow us to build on the pattern used in OSA to provide multi-distro capabilities. A side effect of this change is a major improvement in idempotency. The playbooks should now be 100% idempotent. All of the templates have been left in the main playbook directory. This was done to help ease the transition. In a future PR the template structure will be moved into the roles where it needs to be. The main variable files has been left intact. This file will be carved up into role defaults in a future PR. Change-Id: I938a10564128ce4078fa12edcf614dcdbd684b25 Signed-off-by: Kevin Carter --- .../calculate_index_retention_default.yml | 24 -- .../common_task_data_node_hosts.yml | 131 --------- .../common_task_install_go1.10.1.yml | 62 ---- elk_metrics_6x/createElasticIndexes.yml | 20 +- elk_metrics_6x/installAPMserver.yml | 88 ++---- elk_metrics_6x/installAuditbeat.yml | 85 +----- elk_metrics_6x/installCurator.yml | 126 +------- elk_metrics_6x/installElastic.yml | 242 +-------------- elk_metrics_6x/installFilebeat.yml | 194 +----------- elk_metrics_6x/installHeartbeat.yml | 80 +---- elk_metrics_6x/installJournalbeat.yml | 145 +-------- elk_metrics_6x/installKibana.yml | 117 +------- elk_metrics_6x/installLogstash.yml | 278 +----------------- elk_metrics_6x/installMetricbeat.yml | 236 +-------------- elk_metrics_6x/installPacketbeat.yml | 85 +----- .../elastic_apm_server/handlers/main.yml | 25 ++ .../roles/elastic_apm_server/meta/main.yml | 33 +++ .../elastic_apm_server/tasks/apm_setup.yml | 30 ++ .../roles/elastic_apm_server/tasks/main.yml | 70 +++++ .../roles/elastic_apm_server/vars/ubuntu.yml | 17 ++ .../roles/elastic_auditbeat/handlers/main.yml | 25 ++ .../roles/elastic_auditbeat/meta/main.yml | 33 +++ .../tasks/auditbeat_setup.yml | 32 ++ .../roles/elastic_auditbeat/tasks/main.yml | 68 +++++ .../roles/elastic_auditbeat/vars/ubuntu.yml | 18 ++ .../roles/elastic_curator/handlers/main.yml | 24 ++ .../roles/elastic_curator/meta/main.yml | 32 ++ .../roles/elastic_curator/tasks/main.yml | 111 +++++++ .../roles/elastic_curator/vars/ubuntu.yml | 18 ++ .../roles/elastic_data_hosts/meta/main.yml | 31 ++ .../roles/elastic_data_hosts/tasks/main.yml | 25 ++ .../vars/data-node-variables.yml | 117 ++++++++ .../elastic_dependencies/defaults/main.yml | 22 ++ .../roles/elastic_dependencies/meta/main.yml | 32 ++ .../roles/elastic_dependencies/tasks/main.yml | 162 ++++++++++ .../elastic_dependencies/vars/ubuntu.yml | 17 ++ .../vars/vars_elasticsearch.yml | 15 + .../vars/vars_logstash.yml | 15 + .../roles/elastic_filebeat/handlers/main.yml | 25 ++ .../roles/elastic_filebeat/meta/main.yml | 33 +++ .../elastic_filebeat/tasks/filebeat_setup.yml | 32 ++ .../roles/elastic_filebeat/tasks/main.yml | 177 +++++++++++ .../roles/elastic_filebeat/vars/ubuntu.yml | 17 ++ .../roles/elastic_heartbeat/handlers/main.yml | 25 ++ .../roles/elastic_heartbeat/meta/main.yml | 33 +++ .../tasks/heartbeat_setup.yml | 32 ++ .../roles/elastic_heartbeat/tasks/main.yml | 63 ++++ .../roles/elastic_heartbeat/vars/ubuntu.yml | 17 ++ .../elastic_journalbeat/handlers/main.yml | 25 ++ .../roles/elastic_journalbeat/meta/main.yml | 33 +++ .../tasks/journalbeat_setup.yml | 40 +++ .../roles/elastic_journalbeat/tasks/main.yml | 134 +++++++++ .../roles/elastic_journalbeat/vars/ubuntu.yml | 19 ++ .../roles/elastic_kibana/defaults/main.yml | 16 + .../roles/elastic_kibana/handlers/main.yml | 37 +++ .../roles/elastic_kibana/meta/main.yml | 32 ++ .../roles/elastic_kibana/tasks/main.yml | 84 ++++++ .../roles/elastic_kibana/vars/ubuntu.yml | 20 ++ .../roles/elastic_logstash/defaults/main.yml | 24 ++ .../roles/elastic_logstash/handlers/main.yml | 25 ++ .../roles/elastic_logstash/meta/main.yml | 37 +++ .../tasks/logstash_arcsight.yml | 62 ++++ .../tasks/logstash_kafka_ssl.yml | 24 ++ .../roles/elastic_logstash/tasks/main.yml | 129 ++++++++ .../roles/elastic_logstash/vars/ubuntu.yml | 18 ++ .../elastic_metricbeat/handlers/main.yml | 25 ++ .../roles/elastic_metricbeat/meta/main.yml | 33 +++ .../roles/elastic_metricbeat/tasks/main.yml | 236 +++++++++++++++ .../tasks/metricbeat_setup.yml | 32 ++ .../roles/elastic_metricbeat/vars/ubuntu.yml | 17 ++ .../elastic_packetbeat/handlers/main.yml | 25 ++ .../roles/elastic_packetbeat/meta/main.yml | 33 +++ .../roles/elastic_packetbeat/tasks/main.yml | 63 ++++ .../tasks/packetbeat_setup.yml | 32 ++ .../roles/elastic_packetbeat/vars/ubuntu.yml | 18 ++ .../roles/elastic_repositories/meta/main.yml | 31 ++ .../tasks/elastic_apt_repos.yml} | 12 - .../roles/elastic_repositories/tasks/main.yml | 40 +++ .../elastic_repositories/vars/ubuntu.yml | 17 ++ .../roles/elastic_retention/defaults/main.yml | 16 + .../roles/elastic_retention/meta/main.yml | 32 ++ .../roles/elastic_retention/tasks/main.yml | 35 +++ .../calculate_index_retention_default.yml | 30 ++ .../roles/elastic_rollup/defaults/main.yml | 16 + .../roles/elastic_rollup/meta/main.yml | 32 ++ .../roles/elastic_rollup/tasks/main.yml | 96 ++++++ .../roles/elasticsearch/defaults/main.yml | 19 ++ .../roles/elasticsearch/handlers/main.yml | 25 ++ .../roles/elasticsearch/meta/main.yml | 37 +++ .../tasks/elasticsearch_nfs_setup.yml | 29 ++ .../roles/elasticsearch/tasks/main.yml | 102 +++++++ .../roles/elasticsearch/vars/ubuntu.yml | 18 ++ .../roles/elasticsearch/vars/vars_default.yml | 17 ++ .../roles/elasticsearch/vars/vars_kibana.yml | 22 ++ .../roles/go_install_1.10/handlers/main.yml | 14 + .../roles/go_install_1.10/meta/main.yml | 31 ++ .../go_install_1.10/tasks/go_install.yml | 45 +++ .../roles/go_install_1.10/tasks/main.yml | 23 ++ elk_metrics_6x/setupAPMserver.yml | 31 -- elk_metrics_6x/setupAuditbeat.yml | 31 -- elk_metrics_6x/setupFilebeat.yml | 31 -- elk_metrics_6x/setupHeartbeat.yml | 31 -- elk_metrics_6x/setupJournalbeat.yml | 47 --- elk_metrics_6x/setupKibanaDashboard.yml | 12 + elk_metrics_6x/setupMetricbeat.yml | 31 -- elk_metrics_6x/setupPacketbeat.yml | 31 -- elk_metrics_6x/showElasticCluster.yml | 18 +- elk_metrics_6x/site-beats-community.yml | 2 - elk_metrics_6x/site-beats-core.yml | 2 - elk_metrics_6x/site-elka.yml | 2 - elk_metrics_6x/site.yml | 2 - elk_metrics_6x/templates/jvm.options.j2 | 4 +- elk_metrics_6x/tests/testLayout.yml | 11 + 113 files changed, 3626 insertions(+), 2011 deletions(-) delete mode 100644 elk_metrics_6x/calculate_index_retention_default.yml delete mode 100644 elk_metrics_6x/common_task_data_node_hosts.yml delete mode 100644 elk_metrics_6x/common_task_install_go1.10.1.yml create mode 100644 elk_metrics_6x/roles/elastic_apm_server/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elastic_apm_server/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml create mode 100644 elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_apm_server/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_auditbeat/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elastic_auditbeat/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml create mode 100644 elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_auditbeat/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_curator/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elastic_curator/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_curator/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_curator/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_data_hosts/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_data_hosts/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml create mode 100644 elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml create mode 100644 elk_metrics_6x/roles/elastic_dependencies/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_dependencies/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml create mode 100644 elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml create mode 100644 elk_metrics_6x/roles/elastic_filebeat/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elastic_filebeat/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml create mode 100644 elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_filebeat/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_heartbeat/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elastic_heartbeat/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml create mode 100644 elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_heartbeat/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_journalbeat/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml create mode 100644 elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_kibana/defaults/main.yml create mode 100644 elk_metrics_6x/roles/elastic_kibana/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elastic_kibana/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_kibana/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_kibana/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_logstash/defaults/main.yml create mode 100644 elk_metrics_6x/roles/elastic_logstash/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elastic_logstash/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_logstash/tasks/logstash_arcsight.yml create mode 100644 elk_metrics_6x/roles/elastic_logstash/tasks/logstash_kafka_ssl.yml create mode 100644 elk_metrics_6x/roles/elastic_logstash/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_logstash/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_metricbeat/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elastic_metricbeat/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml create mode 100644 elk_metrics_6x/roles/elastic_metricbeat/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_packetbeat/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elastic_packetbeat/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml create mode 100644 elk_metrics_6x/roles/elastic_packetbeat/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_repositories/meta/main.yml rename elk_metrics_6x/{common_task_install_elk_repo.yml => roles/elastic_repositories/tasks/elastic_apt_repos.yml} (84%) create mode 100644 elk_metrics_6x/roles/elastic_repositories/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_repositories/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elastic_retention/defaults/main.yml create mode 100644 elk_metrics_6x/roles/elastic_retention/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_retention/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elastic_retention/vars/calculate_index_retention_default.yml create mode 100644 elk_metrics_6x/roles/elastic_rollup/defaults/main.yml create mode 100644 elk_metrics_6x/roles/elastic_rollup/meta/main.yml create mode 100644 elk_metrics_6x/roles/elastic_rollup/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elasticsearch/defaults/main.yml create mode 100644 elk_metrics_6x/roles/elasticsearch/handlers/main.yml create mode 100644 elk_metrics_6x/roles/elasticsearch/meta/main.yml create mode 100644 elk_metrics_6x/roles/elasticsearch/tasks/elasticsearch_nfs_setup.yml create mode 100644 elk_metrics_6x/roles/elasticsearch/tasks/main.yml create mode 100644 elk_metrics_6x/roles/elasticsearch/vars/ubuntu.yml create mode 100644 elk_metrics_6x/roles/elasticsearch/vars/vars_default.yml create mode 100644 elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml create mode 100644 elk_metrics_6x/roles/go_install_1.10/handlers/main.yml create mode 100644 elk_metrics_6x/roles/go_install_1.10/meta/main.yml create mode 100644 elk_metrics_6x/roles/go_install_1.10/tasks/go_install.yml create mode 100644 elk_metrics_6x/roles/go_install_1.10/tasks/main.yml delete mode 100644 elk_metrics_6x/setupAPMserver.yml delete mode 100644 elk_metrics_6x/setupAuditbeat.yml delete mode 100644 elk_metrics_6x/setupFilebeat.yml delete mode 100644 elk_metrics_6x/setupHeartbeat.yml delete mode 100644 elk_metrics_6x/setupJournalbeat.yml delete mode 100644 elk_metrics_6x/setupMetricbeat.yml delete mode 100644 elk_metrics_6x/setupPacketbeat.yml diff --git a/elk_metrics_6x/calculate_index_retention_default.yml b/elk_metrics_6x/calculate_index_retention_default.yml deleted file mode 100644 index 058cfbf2..00000000 --- a/elk_metrics_6x/calculate_index_retention_default.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- - -- name: Set assumed buffer storage fact - set_fact: - es_assumed_buffer_storage: "{{ ((es_total_available_storage | int) * 0.25) | round | int }}" - -- name: Set usable buffer storage fact(s) - set_fact: - es_usable_buffer_storage: "{{ (es_total_available_storage | int) - (es_assumed_buffer_storage | int) }}" - es_expected_storage: "{{ ((elastic_beat_retention_policy_hosts.values() | map('int') | list) | sum) * (elastic_beat_storage_constant | int) }}" - -- name: Set buffer storage fact - set_fact: - es_assumed_usable_storage_per_node: "{{ (es_usable_buffer_storage | int) // (data_nodes | length | int) }}" - -- name: Set storage the mulitplier - set_fact: - es_storage_multiplier: "{{ ((es_usable_buffer_storage | int) < (es_expected_storage | int)) | ternary(((elastic_beat_storage_constant | int) * 2), elastic_beat_storage_constant | int) }}" - -- name: Set retention facts - set_fact: "elastic_{{ item.key }}_retention={{ (es_assumed_usable_storage_per_node | int) // ((item.value | int) * (es_storage_multiplier | int)) }}" - when: - - hostvars[inventory_hostname]["elastic_" + item.key + "_retention"] is undefined - with_dict: "{{ elastic_beat_retention_policy_hosts }}" diff --git a/elk_metrics_6x/common_task_data_node_hosts.yml b/elk_metrics_6x/common_task_data_node_hosts.yml deleted file mode 100644 index 98f87508..00000000 --- a/elk_metrics_6x/common_task_data_node_hosts.yml +++ /dev/null @@ -1,131 +0,0 @@ ---- - -- name: Refresh minimal facts - setup: - gather_subset: '!all,!any,network,virtual' - -# storage node count is equal to the cluster size -- name: Node count fact - set_fact: - storage_node_count: "{{ groups['elastic-logstash'] | length }}" - tags: - - always - -# the elasticserch cluster elects one master from all those which are marked as master-eligible -# 1 node cluster can only have one master -# 2 node clusters have 1 master-eligable nodes to avoid split-brain -# 3 node clusters have 3 master-eligable nodes -# >3 node clusters have (nodes // 2) eligable masters rounded up to the next odd number -- name: Master node count fact - set_fact: - master_node_count: |- - {% set masters = 0 %} - {% if (storage_node_count | int) < 3 %} - {% set masters = 1 %} - {% elif (storage_node_count | int) == 3 %} - {% set masters = 3 %} - {% else %} - {% set masters = (storage_node_count | int ) // 2 %} - {% if ((masters | int) % 2 == 0) %} - {% set masters = (masters | int) + 1 %} - {% endif %} - {% endif %} - {{ masters }} - tags: - - always - -# assign node roles -# the first 'master_node_count' hosts in groups['elastic-logstash'] become master-eligible nodes -# the first 'master_node_count' and subsequent alternate hosts in groups['elastic-logstash'] becomes data nodes -- name: Data nodes fact - set_fact: - data_nodes: "{{ (groups['elastic-logstash'][:master_node_count | int] + groups['elastic-logstash'][master_node_count | int::2]) }}" - master_nodes: "{{ groups['elastic-logstash'][:master_node_count | int] }}" - coordination_nodes: >- - {{ - (groups['kibana'] | map('extract', hostvars, 'ansible_host') | list) - | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) - | list - }} - zen_nodes: >- - {{ - (groups['elastic-logstash'] | union(groups['kibana'])) | map('extract', hostvars, 'ansible_host') | list - }} - elasticserch_interface_speed: |- - {% set default_interface_fact = hostvars[inventory_hostname]['ansible_' + (elastic_data_interface | replace('-', '_'))] %} - {% set speeds = [] %} - {% if default_interface_fact['type'] == 'bridge' %} - {% for interface in default_interface_fact['interfaces'] %} - {% set interface_fact = hostvars[inventory_hostname]['ansible_' + (interface | replace('-', '_'))] %} - {% if 'speed' in interface_fact %} - {% set speed = (interface_fact['speed'] | default(1000)) | string %} - {% if speed == "-1" %} - {% set _ = speeds.append(1000) %} - {% else %} - {% set _ = speeds.append(speed | int) %} - {% endif %} - {% if 'module' in interface_fact %} - {% set _ = speeds.append((interface_fact['speed'] | default(1000)) | int) %} - {% else %} - {% set _ = speeds.append(1000) %} - {% endif %} - {% endif %} - {% endfor %} - {% else %} - {% if ('module' in default_interface_fact) or (default_interface_fact['type'] == 'bond') %} - {% set speed = (default_interface_fact['speed'] | default(1000)) | string %} - {% if speed == "-1" %} - {% set _ = speeds.append(1000) %} - {% else %} - {% set _ = speeds.append(speed | int) %} - {% endif %} - {% else %} - {% set _ = speeds.append(1000) %} - {% endif %} - {% endif %} - {% set interface_speed = ((speeds | min) * 0.20) | int %} - {{ ((interface_speed | int) > 750) | ternary(750, interface_speed) }} - tags: - - always - -- name: Set data node details - set_fact: - elasticsearch_data_node_details: >- - {{ - (data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) | list - }} - logstash_data_node_details: >- - {{ - (data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ logstash_beat_input_port) | list - }} - -# based on the assignment of roles to hosts, set per host booleans -- name: Node enablement - set_fact: - master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}" - data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}" - elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) >= 24) | ternary(24, ansible_processor_cores) }}" - tags: - - always - -# Set a data node facts. The data nodes, in the case of elasticsearch are also -# ingest nodes. -- name: Set data nodes - set_fact: - elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 2) | ternary('2', ((data_nodes | length) > 1) | ternary('1', '0')) }}" - elasticsearch_data_hosts: |- - {% set nodes = elasticsearch_data_node_details %} - {% if inventory_hostname in data_nodes %} - {% set _ = nodes.insert(0, '127.0.0.1:' ~ elastic_port) %} - {% endif %} - {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} - {{ data_hosts }} - logstash_data_hosts: |- - {% set nodes = logstash_data_node_details %} - {% if inventory_hostname in data_nodes %} - {% set _ = nodes.insert(0, '127.0.0.1:' ~ logstash_beat_input_port) %} - {% endif %} - {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} - {{ data_hosts }} - tags: - - always diff --git a/elk_metrics_6x/common_task_install_go1.10.1.yml b/elk_metrics_6x/common_task_install_go1.10.1.yml deleted file mode 100644 index 4d389438..00000000 --- a/elk_metrics_6x/common_task_install_go1.10.1.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Check for go - stat: - path: /opt/go1.10.1/go/bin/go - register: go_path - -- name: Install go - block: - - name: GET go - get_url: - url: "https://dl.google.com/go/go1.10.1.linux-amd64.tar.gz" - dest: "/opt/go1.10.1.linux-amd64.tar.gz" - checksum: "sha256:72d820dec546752e5a8303b33b009079c15c2390ce76d67cf514991646c6127b" - register: _get_task - until: _get_task is success - retries: 3 - delay: 2 - tags: - - package_install - - - name: Create go directory - file: - path: "/opt/go1.10.1" - state: directory - - - name: Unarchive go - unarchive: - src: "/opt/go1.10.1.linux-amd64.tar.gz" - dest: "/opt/go1.10.1" - remote_src: yes - - - name: Create go defaults file - copy: - content: | - GOROOT=/opt/go1.10.1/go - GOPATH=/usr/local - PATH=${PATH}:${GOROOT}/bin - dest: /etc/default/go1.10.1 - when: - - (elk_package_state | default('present')) == 'present' - - not go_path.stat.exists | bool - -- name: Remove go - file: - path: "/opt/go1.10.1" - state: absent - when: - - (elk_package_state | default('present')) == 'absent' diff --git a/elk_metrics_6x/createElasticIndexes.yml b/elk_metrics_6x/createElasticIndexes.yml index b05f3ac8..2b7c45c1 100644 --- a/elk_metrics_6x/createElasticIndexes.yml +++ b/elk_metrics_6x/createElasticIndexes.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Create/Setup known indexes in Elasticsearch hosts: "elastic-logstash[0]" become: true @@ -8,12 +20,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always + roles: + - role: elastic_retention - tasks: + post_tasks: - name: Create basic indexes uri: url: http://127.0.0.1:9200/{{ item.name }} diff --git a/elk_metrics_6x/installAPMserver.yml b/elk_metrics_6x/installAPMserver.yml index 137fc8d9..634c2038 100644 --- a/elk_metrics_6x/installAPMserver.yml +++ b/elk_metrics_6x/installAPMserver.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install apm-server hosts: apm-server become: true @@ -8,76 +20,10 @@ vars_files: - vars/variables.yml - environment: "{{ deployment_environment_variables | default({}) }}" - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure apm-server is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - with_items: - - apm-server - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart apm server - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create apm-server systemd service config dir - file: - path: "/etc/systemd/system/apm-server.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/apm-server.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "apm-server-overrides.conf" } - notify: - - Enable and restart apm server - - - name: Drop apm-server conf file - template: - src: templates/apm-server.yml.j2 - dest: /etc/apm-server/apm-server.yml - notify: - - Enable and restart apm server - - handlers: - - name: Enable and restart apm server - systemd: - name: "apm-server" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_apm_server + - role: elastic_rollup + index_name: apm tags: - - server-install - -- import_playbook: setupAPMserver.yml + - apm-server diff --git a/elk_metrics_6x/installAuditbeat.yml b/elk_metrics_6x/installAuditbeat.yml index 6f69d25f..121e94f4 100644 --- a/elk_metrics_6x/installAuditbeat.yml +++ b/elk_metrics_6x/installAuditbeat.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Auditbeat hosts: hosts become: true @@ -10,75 +22,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure Auditbeat is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - with_items: - - audispd-plugins - - auditbeat - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart auditbeat - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create auditbeat systemd service config dir - file: - path: "/etc/systemd/system/auditbeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/auditbeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "auditbeat-overrides.conf" } - notify: - - Enable and restart auditbeat - - - name: Drop auditbeat conf file - template: - src: templates/auditbeat.yml.j2 - dest: /etc/auditbeat/auditbeat.yml - notify: - - Enable and restart auditbeat - - handlers: - - name: Enable and restart auditbeat - systemd: - name: "auditbeat" - enabled: "true" - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_auditbeat + - role: elastic_rollup + index_name: auditbeat tags: - beat-install - -- import_playbook: setupAuditbeat.yml diff --git a/elk_metrics_6x/installCurator.yml b/elk_metrics_6x/installCurator.yml index 221461b5..85746723 100644 --- a/elk_metrics_6x/installCurator.yml +++ b/elk_metrics_6x/installCurator.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -25,128 +23,8 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - - name: Query es storage - uri: - url: "http://127.0.0.1:9200/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs" - method: GET - register: elk_data - until: elk_data is success - retries: 5 - delay: 5 - run_once: true - - - name: Set available storage fact - set_fact: - es_total_available_storage: "{{ ((elk_data['json']['nodes'].values() | list) | map(attribute='fs.total.total_in_bytes') | list | sum) // 1024 // 1024 }}" - - - name: Compute index retention - include_tasks: "calculate_index_retention_{{ elastic_index_retention_algorithm | default('default') }}.yml" - - - name: Ensure virtualenv is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - with_items: - - python-virtualenv - - virtualenv - tags: - - package_install - - - name: Ensure curator is installed - pip: - name: "elasticsearch-curator<6" - state: "{{ elk_package_state | default('present') }}" - extra_args: --isolated - virtualenv: /opt/elasticsearch-curator - register: _pip_task - until: _pip_task is success - retries: 3 - delay: 2 - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - tasks: - - name: create the system group - group: - name: "curator" - state: "present" - system: "yes" - - - name: Create the curator system user - user: - name: "curator" - group: "curator" - comment: "curator user" - shell: "/bin/false" - createhome: "yes" - home: "/var/lib/curator" - - - name: Create curator data path - file: - path: "{{ item }}" - state: directory - owner: "curator" - group: "curator" - mode: "0755" - recurse: true - with_items: - - "/var/lib/curator" - - "/var/log/curator" - - "/etc/curator" - - - name: Drop curator conf file - template: - src: templates/curator.yml.j2 - dest: /var/lib/curator/curator.yml - - - name: Drop curator action file - template: - src: templates/curator-actions.yml.j2 - dest: /var/lib/curator/actions.yml - - post_tasks: - - name: Run the systemd service role - include_role: - name: systemd_service - private: true - vars: - systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" - systemd_service_restart_changed: false - systemd_user_name: curator - systemd_group_name: curator - systemd_services: - - service_name: "curator" - execstarts: - - /opt/elasticsearch-curator/bin/curator - --config /var/lib/curator/curator.yml - /var/lib/curator/actions.yml - timer: - state: "started" - options: - OnBootSec: 30min - OnUnitActiveSec: 24h - Persistent: true - - - name: Enable and restart curator.timer - systemd: - name: "curator.timer" - enabled: true - state: restarted - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_curator tags: - beat-install diff --git a/elk_metrics_6x/installElastic.yml b/elk_metrics_6x/installElastic.yml index 46f8b7f1..748834b7 100644 --- a/elk_metrics_6x/installElastic.yml +++ b/elk_metrics_6x/installElastic.yml @@ -1,241 +1,27 @@ --- - -- name: Run serialization detection - hosts: "elastic-logstash:kibana" - gather_facts: true - - vars_files: - - vars/variables.yml - - tasks: - - include_tasks: common_task_data_node_hosts.yml - - - name: Group by stand alone masters - group_by: - key: elastic_masters - parents: elastic-logstash - when: - - inventory_hostname in master_nodes - - - name: Group by non stand alone masters - group_by: - key: elastic_non_masters - parents: elastic-logstash - when: - - inventory_hostname in (data_nodes | difference(master_nodes)) - - - name: Group by coordinators - group_by: - key: elastic_coordinators - parents: elastic-logstash - when: - - inventory_hostname in groups['kibana'] - tags: - - always +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. - name: Install Elastic Search - hosts: "elastic_coordinators:elastic_masters:elastic_non_masters" - serial: "33%" + hosts: "elastic-logstash:kibana" become: true vars_files: - vars/variables.yml - vars: - temp_dir: /var/lib/elasticsearch/tmp - nfs_query: "[?fstype=='nfs' || fstype=='nfs4']" - environment: "{{ deployment_environment_variables | default({}) }}" - tasks: - - name: Set memory fact to half - set_fact: - h_mem: "{{ (ansible_memtotal_mb | int) // 2 }}" - when: - - h_mem is not defined - tags: - - always - - - name: Set elasticsearch facts - set_fact: - elastic_heap_size: "{{ ((h_mem | int) > 30720) | ternary(30720, h_mem) }}" - elastic_log_rotate_path: "/var/log/elasticsearch" - tags: - - always - - - name: Set kibana elasticsearch facts - block: - - name: Set kibana as elasticsearch coordinators - set_fact: - elasticsearch_node_master: false - elasticsearch_node_data: false - elasticsearch_node_ingest: false - elastic_coordination_node: true - elastic_heap_size: "{{ (elastic_heap_size | int) // 3 }}" - elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 4) | ternary(4, (ansible_processor_cores // 2)) }}" - when: - - inventory_hostname in (groups['kibana'] | default([])) and - not inventory_hostname in (groups['elastic-logstash'] | default([])) - tags: - - always - - - name: Configure systcl vm.max_map_count=262144 on container hosts - sysctl: - name: "vm.max_map_count" - value: "262144" - state: "present" - reload: "yes" - delegate_to: "{{ physical_host }}" - tags: - - sysctl - - - name: Ensure mount directories exists - file: - path: "/openstack/{{ inventory_hostname }}/elasticsearch" - state: "directory" - delegate_to: "{{ physical_host }}" - - - name: elasticsearch datapath bind mount - lxc_container: - name: "{{ inventory_hostname }}" - container_command: | - [[ ! -d "/var/lib/elasticsearch" ]] && mkdir -p "/var/lib/elasticsearch" - container_config: - - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/elasticsearch var/lib/elasticsearch none bind 0 0" - delegate_to: "{{ physical_host }}" - when: - - physical_host != inventory_hostname - - container_tech | default('lxc') == 'lxc' - - - name: Ensure Java is installed - apt: - name: openjdk-8-jre - state: "{{ elk_package_state | default('present') }}" - install_recommends: yes - update_cache: yes - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - tags: - - package_install - - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure elasticsearch is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: yes - with_items: - - logrotate - - elasticsearch - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart elastic - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create elasticsearch systemd service config dir - file: - path: "/etc/systemd/system/elasticsearch.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/elasticsearch.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.elasticsearch-overrides.conf.j2", dest: "elasticsearch-overrides.conf" } - notify: - - Enable and restart elastic - - - name: Drop elasticsearch conf file - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - with_items: - - src: templates/elasticsearch.yml.j2 - dest: /etc/elasticsearch/elasticsearch.yml - - src: templates/jvm.options.j2 - dest: /etc/elasticsearch/jvm.options - - src: templates/es-log4j2.properties.j2 - dest: /etc/elasticsearch/log4j2.properties - notify: - - Enable and restart elastic - tags: - - config - - - name: Ensure elasticsearch ownership - file: - path: /var/lib/elasticsearch - owner: elasticsearch - group: elasticsearch - recurse: true - register: e_perms - until: e_perms is success - retries: 3 - delay: 1 - tags: - - config - - - name: Ensure logstash tmp dir - file: - path: "/var/lib/elasticsearch/tmp" - state: directory - owner: "elasticsearch" - group: "elasticsearch" - mode: "0750" - - - name: Create logrotate config - template: - src: "templates/logrotate.j2" - dest: "/etc/logrotate.d/elasticsearch" - - - name: Ensure nfs client is available if necessary - package: - name: nfs-common - state: present - when: - - elastic_shared_fs_repos is defined - - (elastic_shared_fs_repos|json_query(nfs_query)) | length > 0 - - - name: Ensure backup filesystems are mounted - mount: - fstype: "{{ item.fstype }}" - src: "{{ item.src }}" - opts: "{{ item.opts }}" - path: "{{ item.path }}" - state: "{{ item.state }}" - with_items: - - "{{ elastic_shared_fs_repos }}" - when: - - elastic_shared_fs_repos is defined and elastic_shared_fs_repos | length > 0 - - handlers: - - name: Enable and restart elastic - systemd: - name: "elasticsearch" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elasticsearch tags: - server-install diff --git a/elk_metrics_6x/installFilebeat.yml b/elk_metrics_6x/installFilebeat.yml index 16733fc3..5b349fde 100644 --- a/elk_metrics_6x/installFilebeat.yml +++ b/elk_metrics_6x/installFilebeat.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Filebeat hosts: hosts become: true @@ -10,184 +22,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure Filebeat is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - with_items: - - filebeat - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart filebeat - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create filebeat systemd service config dir - file: - path: "/etc/systemd/system/filebeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/filebeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "filebeat-overrides.conf" } - notify: - - Enable and restart filebeat - - - name: Check for apache - stat: - path: /etc/apache2 - register: apache2 - - - name: Check for auditd - stat: - path: /etc/audit - register: audit - - - name: Check for ceph - stat: - path: /var/log/ceph - register: ceph - - - name: Check for cinder - stat: - path: /var/log/cinder - register: cinder - - - name: Check for glance - stat: - path: /var/log/glance - register: glance - - - name: Check for heat - stat: - path: /var/log/heat - register: heat - - - name: Check for horizon - stat: - path: /var/log/horizon - register: horizon - - - name: Check for httpd - stat: - path: /var/log/httpd - register: httpd - - - name: Check for keystone - stat: - path: /var/log/keystone - register: keystone - - - name: Check for mysql - stat: - path: /var/lib/mysql - register: mysql - - - name: Check for neutron - stat: - path: /var/log/neutron - register: neutron - - - name: Check for nginx - stat: - path: /var/log/nginx - register: nginx - - - name: Check for nova - stat: - path: /var/log/nova - register: nova - - - name: Check for octavia - stat: - path: /var/log/octavia - register: octavia - - - name: Check for swift - stat: - path: /var/log/swift - register: swift - - - name: Check for rabbitmq - stat: - path: /var/lib/rabbitmq - register: rabbitmq - - - name: Check for designate - stat: - path: /var/log/designate - register: designate - - - name: Check for osquery - stat: - path: /var/log/osquery/osqueryd.results.log - register: osquery - - - name: Set discovery facts - set_fact: - apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}" - nginx_enabled: "{{ nginx.stat.exists | bool }}" - auditd_enabled: "{{ audit.stat.exists | bool }}" - mysql_enabled: "{{ (mysql.stat.exists | bool) or (inventory_hostname in groups['galera_all'] | default([])) }}" - ceph_enabled: "{{ (ceph.stat.exists | bool) or (inventory_hostname in groups['ceph_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*ceph.*') | list | length) > 0) }}" - cinder_enabled: "{{ (cinder.stat.exists | bool) or (inventory_hostname in groups['cinder_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*cinder.*') | list | length) > 0) }}" - glance_enabled: "{{ (glance.stat.exists | bool) or (inventory_hostname in groups['glance_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*glance.*') | list | length) > 0) }}" - heat_enabled: "{{ (heat.stat.exists | bool) or (inventory_hostname in groups['heat_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*heat.*') | list | length) > 0) }}" - horizon_enabled: "{{ (horizon.stat.exists | bool) or (inventory_hostname in groups['horizon_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*horizon.*') | list | length) > 0) }}" - keystone_enabled: "{{ (keystone.stat.exists | bool) or (inventory_hostname in groups['keystone_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*keystone.*') | list | length) > 0) }}" - neutron_enabled: "{{ (neutron.stat.exists | bool) or (inventory_hostname in groups['neutron_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*neutron.*') | list | length) > 0) }}" - nova_enabled: "{{ (nova.stat.exists | bool) or (inventory_hostname in groups['nova_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*nova.*') | list | length) > 0) }}" - octavia_enabled: "{{ (octavia.stat.exists | bool) or (inventory_hostname in groups['octavia_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*octavia.*') | list | length) > 0) }}" - swift_enabled: "{{ (swift.stat.exists | bool) or (inventory_hostname in groups['swift_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*swift.*') | list | length) > 0) }}" - rabbitmq_enabled: "{{ (rabbitmq.stat.exists | bool) or (inventory_hostname in groups['rabbitmq_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*rabbit.*') | list | length) > 0) }}" - designate_enabled: "{{ (designate.stat.exists | bool) or (inventory_hostname in groups['designate_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*designate.*') | list | length) > 0) }}" - osquery_enabled: "{{ osquery.stat.exists | bool }}" - - - name: Drop Filebeat conf file - template: - src: templates/filebeat.yml.j2 - dest: /etc/filebeat/filebeat.yml - notify: - - Enable and restart filebeat - - handlers: - - name: Enable and restart filebeat - systemd: - name: "filebeat" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_filebeat + - role: elastic_rollup + index_name: filebeat tags: - beat-install - -- import_playbook: setupFilebeat.yml diff --git a/elk_metrics_6x/installHeartbeat.yml b/elk_metrics_6x/installHeartbeat.yml index 4a4cf0bd..593458b3 100644 --- a/elk_metrics_6x/installHeartbeat.yml +++ b/elk_metrics_6x/installHeartbeat.yml @@ -1,4 +1,15 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. - name: Set heartbeat host deployment group hosts: kibana @@ -11,7 +22,6 @@ parents: kibana when: - inventory_hostname in groups['kibana'][:3] - tags: - always @@ -26,70 +36,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure heartbeat is installed - apt: - name: "heartbeat-elastic" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create heartbeat systemd service config dir - file: - path: "/etc/systemd/system/heartbeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/heartbeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "heartbeat-overrides.conf" } - notify: - - Enable and restart heartbeat - - - name: Drop heartbeat conf file - template: - src: templates/heartbeat.yml.j2 - dest: /etc/heartbeat/heartbeat.yml - notify: - - Enable and restart heartbeat - - handlers: - - name: Enable and restart heartbeat - systemd: - name: "heartbeat-elastic" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_heartbeat + - role: elastic_rollup + index_name: heartbeat tags: - beat-install - -- import_playbook: setupHeartbeat.yml diff --git a/elk_metrics_6x/installJournalbeat.yml b/elk_metrics_6x/installJournalbeat.yml index db71546b..836b3de4 100644 --- a/elk_metrics_6x/installJournalbeat.yml +++ b/elk_metrics_6x/installJournalbeat.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -33,7 +31,7 @@ key: journalbeat_deployment_hosts parents: all_journalbeat_deployments when: - - physical_host is defined and + - physical_host is undefined or physical_host == inventory_hostname tags: @@ -50,143 +48,10 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_go1.10.1.yml - - - name: Check for journal directory - stat: - path: /var/log/journal - register: journal_dir - - - name: Ensure libsystemd-dev is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - with_items: - - gcc - - git - - libsystemd-dev - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create journalbeat systemd service config dir - file: - path: "/etc/systemd/system/journalbeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/journalbeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "journalbeat-overrides.conf" } - notify: - - Enable and restart journalbeat - - - name: create the system group - group: - name: "journalbeat" - state: "present" - system: "yes" - - - name: Create the journalbeat user - user: - name: "journalbeat" - group: "journalbeat" - comment: "journalbeat user" - shell: "/bin/false" - createhome: "yes" - home: "/usr/share/journalbeat" - - - name: Create journalbeat data path - file: - path: "{{ item }}" - state: directory - owner: "journalbeat" - group: "journalbeat" - mode: "0755" - with_items: - - "/usr/share/journalbeat" - - "/var/lib/journalbeat" - - "/etc/journalbeat" - - - name: Install journalbeat - shell: /opt/go1.10.1/go/bin/go get -v github.com/mheese/journalbeat - args: - creates: /usr/local/bin/journalbeat - environment: - PATH: "{{ ansible_env.PATH }}:/opt/go1.10.1/go/bin" - GOPATH: /usr/local - notify: - - Enable and restart journalbeat - - - name: Drop journalbeat conf file - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - with_items: - - src: templates/journalbeat.yml.j2 - dest: /etc/journalbeat/journalbeat.yml - notify: - - Enable and restart journalbeat - - - name: Run the systemd service role - include_role: - name: systemd_service - private: true - vars: - systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" - systemd_service_restart_changed: false - systemd_services: - - service_name: "journalbeat" - execstarts: - - /usr/local/bin/journalbeat - -c /etc/journalbeat/journalbeat.yml - -path.home /usr/share/journalbeat - -path.config /etc/journalbeat - -path.data /var/lib/journalbeat - -path.logs /var/log/beats - config_overrides: - Service: - EnvironmentFile: "-/etc/default/go1.10.1" - Unit: - Documentation: https://github.com/mheese/journalbeat/blob/master/README.md - Wants: network-online.target - After: network-online.target - - handlers: - - name: Enable and restart journalbeat - systemd: - name: "journalbeat" - enabled: true - state: restarted - daemon_reload: yes - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_journalbeat + - role: elastic_rollup + index_name: journalbeat tags: - beat-install - -- import_playbook: setupJournalbeat.yml diff --git a/elk_metrics_6x/installKibana.yml b/elk_metrics_6x/installKibana.yml index 3dc1ee7c..087160c1 100644 --- a/elk_metrics_6x/installKibana.yml +++ b/elk_metrics_6x/installKibana.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Kibana hosts: kibana become: true @@ -7,109 +19,8 @@ environment: "{{ deployment_environment_variables | default({}) }}" - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure Nginx is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: yes - with_items: - - nginx - - apache2-utils - - python-passlib - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart nginx - tags: - - package_install - - - name: create kibana user to access web interface - htpasswd: - path: "/etc/nginx/htpasswd.users" - name: "{{ kibana_username }}" - password: "{{ kibana_password }}" - owner: root - mode: 0644 - - - name: Drop Nginx default conf file - template: - src: templates/nginx_default.j2 - dest: /etc/nginx/sites-available/default - notify: - - Enable and restart nginx - - - name: Ensure kibana is installed - apt: - name: kibana - state: "{{ elk_package_state | default('present') }}" - update_cache: yes - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart kibana - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create kibana systemd service config dir - file: - path: "/etc/systemd/system/kibana.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/kibana.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "kibana-overrides.conf" } - notify: - - Enable and restart kibana - - - name: Drop kibana conf file - template: - src: templates/kibana.yml.j2 - dest: /etc/kibana/kibana.yml - mode: "0666" - notify: - - Enable and restart kibana - - handlers: - - name: Enable and restart kibana - systemd: - name: "kibana" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config - - - name: Enable and restart nginx - systemd: - name: "nginx" - enabled: true - state: restarted - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_kibana tags: - server-install diff --git a/elk_metrics_6x/installLogstash.yml b/elk_metrics_6x/installLogstash.yml index 1c1e128e..02a3e6d9 100644 --- a/elk_metrics_6x/installLogstash.yml +++ b/elk_metrics_6x/installLogstash.yml @@ -1,276 +1,26 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Logstash hosts: elastic-logstash - serial: "50%" become: true vars_files: - vars/variables.yml - vars: - temp_dir: /var/lib/logstash/tmp - logstash_pipelines: "{{lookup('template', 'templates/logstash-pipelines.yml.j2') }}" - environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Set quarter memory fact - set_fact: - q_mem: "{{ (ansible_memtotal_mb | int) // 3 }}" - when: - - q_mem is not defined - tags: - - always - - - name: Set processor cores fact - set_fact: - q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_core | int) * 2 }}" - when: - - q_storage is not defined - tags: - - always - - - name: Set logstash facts - set_fact: - elastic_heap_size: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}" - logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}" - elastic_log_rotate_path: "/var/log/logstash" - tags: - - always - - - include_tasks: common_task_install_elk_repo.yml - - - name: Configure systcl vm.max_map_count=262144 on container hosts - sysctl: - name: "vm.max_map_count" - value: "262144" - state: "present" - reload: "yes" - delegate_to: "{{ physical_host }}" - tags: - - sysctl - - - name: Ensure mount directories exists - file: - path: "/openstack/{{ inventory_hostname }}/logstash" - state: "directory" - delegate_to: "{{ physical_host }}" - - - name: logstash datapath bind mount - lxc_container: - name: "{{ inventory_hostname }}" - container_command: | - [[ ! -d "/var/lib/logstash" ]] && mkdir -p "/var/lib/logstash" - container_config: - - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/logstash var/lib/logstash none bind 0 0" - delegate_to: "{{ physical_host }}" - when: - - physical_host != inventory_hostname - - container_tech | default('lxc') == 'lxc' - - - name: Ensure Java is installed - apt: - name: openjdk-8-jre - state: "{{ elk_package_state | default('present') }}" - install_recommends: yes - update_cache: yes - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - tags: - - package_install - - - name: Ensure Logstash is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: yes - with_items: - - logrotate - - logstash - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart logstash - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create logstash systemd service config dir - file: - path: "/etc/systemd/system/logstash.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/logstash.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "logstash-overrides.conf" } - notify: - - Enable and restart logstash - - - name: Create patterns directory - file: - name: "/opt/logstash/patterns" - owner: "logstash" - group: "logstash" - state: directory - tags: - - logstash-patterns - - - name: Logstash Extra Patterns - template: - src: "{{ item }}" - dest: "/opt/logstash/patterns/{{ item }}" - owner: "logstash" - group: "logstash" - with_items: - - extras - when: - - logstash_deploy_filters - notify: - - Enable and restart logstash - tags: - - logstash-filters - - config - - - name: Run kafka output block - block: - - name: Copy kafka keystore into place - copy: - src: "{{ logstash_kafka_ssl_keystore_location }}" - dest: "/var/lib/logstash/{{ logstash_kafka_ssl_keystore_location | basename }}" - when: - - logstash_kafka_ssl_keystore_location is defined - - - name: Copy kafka truststore into place - copy: - src: "{{ logstash_kafka_ssl_truststore_location }}" - dest: "/var/lib/logstash/{{ logstash_kafka_ssl_truststore_location | basename }}" - when: - - logstash_kafka_ssl_truststore_location is defined - - when: - - logstash_kafka_options is defined - - - name: Drop logstash conf file(s) - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - with_items: - - src: templates/jvm.options.j2 - dest: /etc/logstash/jvm.options - - src: templates/logstash.yml.j2 - dest: /etc/logstash/logstash.yml - - src: templates/logstash-pipelines.yml.j2 - dest: /etc/logstash/pipelines.yml - - src: "templates/logrotate.j2" - dest: "/etc/logrotate.d/logstash" - notify: - - Enable and restart logstash - tags: - - config - - - name: Ensure logstash ownership - file: - path: /var/lib/logstash - owner: logstash - group: logstash - recurse: true - register: l_perms - until: l_perms is success - retries: 3 - delay: 1 - - - name: Ensure logstash tmp dir - file: - path: "/var/lib/logstash/tmp" - state: directory - owner: "logstash" - group: "logstash" - mode: "0750" - - - name: Run arcsight output block - block: - - name: Initialise arcsight local facts - ini_file: - dest: "/etc/ansible/facts.d/logstash.fact" - section: arcsight - option: "initialise" - value: true - - - name: refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - nova-config - - - name: Setup arcsight smart connector - shell: >- - /usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }} - -M "arcsight.var.input.smartconnector.bootstrap_servers={{ item.host }}:{{ item.port }}" - -M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}" - -M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}" - with_items: "{{ logstash_arcsight_smart_connectors }}" - run_once: true - register: smart_connector - until: smart_connector is success - retries: 5 - delay: 5 - - - name: Setup arcsight event broker - shell: >- - /usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }} - -M "arcsight.var.input.eventbroker.bootstrap_servers={{ item.host }}:{{ item.port }}" - -M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}" - -M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}" - with_items: "{{ logstash_arcsight_event_brokers }}" - run_once: true - register: event_broker - until: event_broker is success - retries: 5 - delay: 5 - - - name: Set arcsight local facts - ini_file: - dest: "/etc/ansible/facts.d/logstash.fact" - section: arcsight - option: "{{ item.host }}" - value: "{{ item.port }}" - with_items: "{{ logstash_arcsight_smart_connectors | union(logstash_arcsight_event_brokers) }}" - when: - - logstash_arcsight_smart_connectors or - logstash_arcsight_event_brokers - - handlers: - - name: Enable and restart logstash - systemd: - name: "logstash" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_logstash tags: - server-install diff --git a/elk_metrics_6x/installMetricbeat.yml b/elk_metrics_6x/installMetricbeat.yml index 64629568..d174c1ec 100644 --- a/elk_metrics_6x/installMetricbeat.yml +++ b/elk_metrics_6x/installMetricbeat.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Metricsbeat hosts: all become: true @@ -10,226 +22,10 @@ vars_files: - vars/variables.yml - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure Metricsbeat is installed - apt: - name: metricbeat - state: "{{ elk_package_state | default('present') }}" - update_cache: true - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart metricbeat - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create metricbeat systemd service config dir - file: - path: "/etc/systemd/system/metricbeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/metricbeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "metricbeat-overrides.conf" } - notify: - - Enable and restart metricbeat - - - name: Check for apache - stat: - path: /etc/apache2/sites-available - register: apache2 - - - name: Check for ceph - stat: - path: /etc/ceph - register: ceph - - # gather ceph stats from localhost - # except when a list of mons is provided - - name: Set ceph stats hosts - set_fact: - ceph_stats_hosts: |- - {% set ceph_stats = [] %} - {% if (ceph_mons is defined) and (ceph_mons | length > 0) %} - {% for mon in ceph_mons %} - {% set _ = ceph_stats.insert(loop.index, (mon + ":5000")) %} - {% endfor %} - {% else %} - {% set ceph_stats = [ ansible_hostname + ":5000" ] %} - {% endif %} - {{ ceph_stats }} - - - name: Check for etcd - stat: - path: /etc/etcd - register: etcd - - - name: Check for docker - stat: - path: /var/run/docker.sock - register: docker - - - name: Check for haproxy - stat: - path: /etc/haproxy - register: haproxy - - - name: Check for httpd - stat: - path: /etc/httpd - register: httpd - - - name: Check for kvm - stat: - path: /var/run/libvirt/libvirt-sock - register: kvm - - - name: Check for memcached - stat: - path: /etc/memcached.conf - register: memcached - - - name: Check for mysql - stat: - path: /var/lib/mysql - register: mysql - - - name: Check for nginx - stat: - path: /etc/nginx/nginx.conf - register: nginx - - - name: Check for rabbitmq - stat: - path: /var/lib/rabbitmq - register: rabbitmq - - - name: Check for uwsgi - stat: - path: /etc/uwsgi - register: uwsgi - - - name: Check for uwsgi stats sockets - find: - paths: /tmp - file_type: any - patterns: '*uwsgi-stats.sock' - register: uwsgi_find_sockets - - - name: Set discovery facts - set_fact: - apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}" - - # enable ceph on: cinder volume hosts when we have a list of ceph mons - # otherwise: all hosts which have /etc/ceph - ceph_enabled: |- - {% set ceph_detect = false %} - {% if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups['cinder_volume']) %} - {% set ceph_detect = true %} - {% else %} - {% set ceph_detect = ceph.stat.exists | bool %} - {% endif %} - {{ ceph_detect }} - - docker_enabled: "{{ docker.stat.exists | bool }}" - etcd_enabled: "{{ etcd.stat.exists | bool }}" - haproxy_enabled: "{{ haproxy.stat.exists | bool }}" - kvm_enabled: "{{ kvm.stat.exists | bool }}" - memcached_enabled: "{{ memcached.stat.exists | bool }}" - mysql_enabled: "{{ mysql.stat.exists | bool }}" - nginx_enabled: "{{ nginx.stat.exists | bool }}" - rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}" - uwsgi_enabled: "{{ uwsgi.stat.exists | bool }}" - uwsgi_sockets: "{{ uwsgi_find_sockets }}" - - # Apache 2 stats enablement - - name: Drop apache2 stats site config - template: - src: apache-status.conf.j2 - dest: /etc/apache2/sites-available/apache-status.conf - when: apache_enabled - - - name: Enable apache2 stats site - file: - src: /etc/apache2/sites-available/apache-status.conf - dest: /etc/apache2/sites-enabled/apache-status.conf - state: link - when: apache_enabled - - - name: Ensure apache2 stats mode is enabled - apache2_module: - name: status - state: present - when: apache_enabled - - - name: Reload apache2 - service: - name: apache2 - state: reloaded - when: apache_enabled - - # NGINX stats enablement - - name: Drop nginx stats site config - template: - src: nginx-status.conf.j2 - dest: /etc/nginx/sites-available/nginx-status.conf - when: nginx_enabled - - - name: Enable nginx stats site - file: - src: /etc/nginx/sites-available/nginx-status.conf - dest: /etc/nginx/sites-enabled/nginx-status.conf - state: link - when: nginx_enabled - - - name: Reload nginx - service: - name: nginx - state: reloaded - when: nginx_enabled - - - name: Drop metricbeat conf file - template: - src: templates/metricbeat.yml.j2 - dest: /etc/metricbeat/metricbeat.yml - notify: - - Enable and restart metricbeat - - handlers: - - name: Enable and restart metricbeat - systemd: - name: "metricbeat" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_metricbeat + - role: elastic_rollup + index_name: metricbeat tags: - beat-install - -- import_playbook: setupMetricbeat.yml diff --git a/elk_metrics_6x/installPacketbeat.yml b/elk_metrics_6x/installPacketbeat.yml index cc721784..15086c38 100644 --- a/elk_metrics_6x/installPacketbeat.yml +++ b/elk_metrics_6x/installPacketbeat.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Packetbeat hosts: hosts become: true @@ -10,75 +22,10 @@ vars_files: - vars/variables.yml - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - include_tasks: common_task_install_elk_repo.yml - - - name: Ensure packetbeat is installed - apt: - name: "{{ item }}" - state: "{{ elk_package_state | default('present') }}" - update_cache: true - with_items: - - tcpdump - - packetbeat - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - notify: - - Enable and restart packetbeat - tags: - - package_install - - - name: exit playbook after uninstall - meta: end_play - when: - - (elk_package_state | default('present')) == 'absent' - - post_tasks: - - name: Create packetbeat systemd service config dir - file: - path: "/etc/systemd/system/packetbeat.service.d" - state: "directory" - group: "root" - owner: "root" - mode: "0755" - - - name: Apply systemd options - template: - src: "{{ item.src }}" - dest: "/etc/systemd/system/packetbeat.service.d/{{ item.dest }}" - mode: "0644" - with_items: - - { src: "systemd.general-overrides.conf.j2", dest: "packetbeat-overrides.conf" } - notify: - - Enable and restart packetbeat - - - name: Drop packetbeat conf file - template: - src: templates/packetbeat.yml.j2 - dest: /etc/packetbeat/packetbeat.yml - notify: - - Enable and restart packetbeat - - handlers: - - name: Enable and restart packetbeat - systemd: - name: "packetbeat" - enabled: true - state: restarted - daemon_reload: true - when: - - (elk_package_state | default('present')) != 'absent' - tags: - - config + roles: + - role: elastic_packetbeat + - role: elastic_rollup + index_name: packetbeat tags: - beat-install - -- import_playbook: setupPacketbeat.yml diff --git a/elk_metrics_6x/roles/elastic_apm_server/handlers/main.yml b/elk_metrics_6x/roles/elastic_apm_server/handlers/main.yml new file mode 100644 index 00000000..69f1a9d5 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart apm server + systemd: + name: "apm-server" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_apm_server/meta/main.yml b/elk_metrics_6x/roles/elastic_apm_server/meta/main.yml new file mode 100644 index 00000000..d24d370c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x apm-server role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml b/elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml new file mode 100644 index 00000000..20c3643f --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + apm-server setup + {{ item }} + -E 'apm-server.host=localhost:8200' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true diff --git a/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml b/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml new file mode 100644 index 00000000..cb249018 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml @@ -0,0 +1,70 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure apm-server is installed + package: + name: "{{ apm_server_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart apm server + tags: + - package_install + +- name: Create apm-server systemd service config dir + file: + path: "/etc/systemd/system/apm-server.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "/etc/systemd/system/apm-server.service.d/{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "apm-server-overrides.conf" + notify: + - Enable and restart apm server + +- name: Drop apm-server conf file + template: + src: templates/apm-server.yml.j2 + dest: /etc/apm-server/apm-server.yml + notify: + - Enable and restart apm server + +- include_tasks: apm_setup.yml + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_apm_server/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_apm_server/vars/ubuntu.yml new file mode 100644 index 00000000..3fac8c52 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_apm_server/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apm_server_distro_packages: + - apm-server diff --git a/elk_metrics_6x/roles/elastic_auditbeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_auditbeat/handlers/main.yml new file mode 100644 index 00000000..8fdf3a74 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart auditbeat + systemd: + name: "auditbeat" + enabled: "true" + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_auditbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_auditbeat/meta/main.yml new file mode 100644 index 00000000..19242323 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x auditbeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml b/elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml new file mode 100644 index 00000000..93542218 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + auditbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml new file mode 100644 index 00000000..44ee4b3b --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml @@ -0,0 +1,68 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure beat is installed + package: + name: "{{ auditbeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart auditbeat + tags: + - package_install + +- name: Create auditbeat systemd service config dir + file: + path: "/etc/systemd/system/auditbeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "/etc/systemd/system/auditbeat.service.d/{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "auditbeat-overrides.conf" + notify: + - Enable and restart auditbeat + +- name: Drop auditbeat conf file + template: + src: templates/auditbeat.yml.j2 + dest: /etc/auditbeat/auditbeat.yml + notify: + - Enable and restart auditbeat + +- include_tasks: auditbeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_auditbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_auditbeat/vars/ubuntu.yml new file mode 100644 index 00000000..59cc129c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_auditbeat/vars/ubuntu.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +auditbeat_distro_packages: + - audispd-plugins + - auditbeat diff --git a/elk_metrics_6x/roles/elastic_curator/handlers/main.yml b/elk_metrics_6x/roles/elastic_curator/handlers/main.yml new file mode 100644 index 00000000..93f43ac3 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_curator/handlers/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart curator.timer + systemd: + name: "curator.timer" + enabled: true + state: restarted + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_curator/meta/main.yml b/elk_metrics_6x/roles/elastic_curator/meta/main.yml new file mode 100644 index 00000000..0dcd0de2 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_curator/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x curator role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_retention diff --git a/elk_metrics_6x/roles/elastic_curator/tasks/main.yml b/elk_metrics_6x/roles/elastic_curator/tasks/main.yml new file mode 100644 index 00000000..0cbccce0 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_curator/tasks/main.yml @@ -0,0 +1,111 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure virtualenv is installed + package: + name: "{{ curator_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + tags: + - package_install + +- name: Ensure curator is installed + pip: + name: "elasticsearch-curator<6" + state: "{{ elk_package_state | default('present') }}" + extra_args: --isolated + virtualenv: /opt/elasticsearch-curator + register: _pip_task + until: _pip_task is success + retries: 3 + delay: 2 + tags: + - package_install + +- name: create the system group + group: + name: "curator" + state: "present" + system: "yes" + +- name: Create the curator system user + user: + name: "curator" + group: "curator" + comment: "curator user" + shell: "/bin/false" + createhome: "yes" + home: "/var/lib/curator" + +- name: Create curator data path + file: + path: "{{ item }}" + state: directory + owner: "curator" + group: "curator" + mode: "0755" + recurse: true + with_items: + - "/var/lib/curator" + - "/var/log/curator" + - "/etc/curator" + +- name: Drop curator conf file + template: + src: templates/curator.yml.j2 + dest: /var/lib/curator/curator.yml + notify: + - Enable and restart curator.timer + +- name: Drop curator action file + template: + src: templates/curator-actions.yml.j2 + dest: /var/lib/curator/actions.yml + notify: + - Enable and restart curator.timer + +- name: Run the systemd service role + include_role: + name: systemd_service + private: true + vars: + systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" + systemd_service_restart_changed: false + systemd_user_name: curator + systemd_group_name: curator + systemd_services: + - service_name: "curator" + execstarts: + - /opt/elasticsearch-curator/bin/curator + --config /var/lib/curator/curator.yml + /var/lib/curator/actions.yml + timer: + state: "started" + options: + OnBootSec: 30min + OnUnitActiveSec: 24h + Persistent: true diff --git a/elk_metrics_6x/roles/elastic_curator/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_curator/vars/ubuntu.yml new file mode 100644 index 00000000..363a2210 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_curator/vars/ubuntu.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +curator_distro_packages: + - python-virtualenv + - virtualenv diff --git a/elk_metrics_6x/roles/elastic_data_hosts/meta/main.yml b/elk_metrics_6x/roles/elastic_data_hosts/meta/main.yml new file mode 100644 index 00000000..6b81d61d --- /dev/null +++ b/elk_metrics_6x/roles/elastic_data_hosts/meta/main.yml @@ -0,0 +1,31 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x data hosts role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: [] diff --git a/elk_metrics_6x/roles/elastic_data_hosts/tasks/main.yml b/elk_metrics_6x/roles/elastic_data_hosts/tasks/main.yml new file mode 100644 index 00000000..a2d80925 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_data_hosts/tasks/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Refresh minimal facts + setup: + gather_subset: '!all,!any,network,virtual' + tags: + - always + +- name: Load data node variables + include_vars: "data-node-variables.yml" + tags: + - always diff --git a/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml b/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml new file mode 100644 index 00000000..936edccb --- /dev/null +++ b/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml @@ -0,0 +1,117 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# storage node count is equal to the cluster size +storage_node_count: "{{ groups['elastic-logstash'] | length }}" + +# the elasticserch cluster elects one master from all those which are marked as master-eligible +# 1 node cluster can only have one master +# 2 node clusters have 1 master-eligable nodes to avoid split-brain +# 3 node clusters have 3 master-eligable nodes +# >3 node clusters have (nodes // 2) eligable masters rounded up to the next odd number +master_node_count: |- + {% set masters = 0 %} + {% if (storage_node_count | int) < 3 %} + {% set masters = 1 %} + {% elif (storage_node_count | int) == 3 %} + {% set masters = 3 %} + {% else %} + {% set masters = (storage_node_count | int ) // 2 %} + {% if ((masters | int) % 2 == 0) %} + {% set masters = (masters | int) + 1 %} + {% endif %} + {% endif %} + {{ masters }} + +# Assign node roles +# the first 'master_node_count' hosts in groups['elastic-logstash'] become master-eligible nodes +# the first 'master_node_count' and subsequent alternate hosts in groups['elastic-logstash'] becomes data nodes +data_nodes: "{{ (groups['elastic-logstash'][:master_node_count | int] + groups['elastic-logstash'][master_node_count | int::2]) }}" +master_nodes: "{{ groups['elastic-logstash'][:master_node_count | int] }}" +coordination_nodes: >- + {{ + (groups['kibana'] | map('extract', hostvars, 'ansible_host') | list) + | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) + | list + }} +zen_nodes: >- + {{ + (groups['elastic-logstash'] | union(groups['kibana'])) | map('extract', hostvars, 'ansible_host') | list + }} +elasticserch_interface_speed: |- + {% set default_interface_fact = hostvars[inventory_hostname]['ansible_' + (elastic_data_interface | replace('-', '_'))] %} + {% set speeds = [] %} + {% if default_interface_fact['type'] == 'bridge' %} + {% for interface in default_interface_fact['interfaces'] %} + {% set interface_fact = hostvars[inventory_hostname]['ansible_' + (interface | replace('-', '_'))] %} + {% if 'speed' in interface_fact %} + {% set speed = (interface_fact['speed'] | default(1000)) | string %} + {% if speed == "-1" %} + {% set _ = speeds.append(1000) %} + {% else %} + {% set _ = speeds.append(speed | int) %} + {% endif %} + {% if 'module' in interface_fact %} + {% set _ = speeds.append((interface_fact['speed'] | default(1000)) | int) %} + {% else %} + {% set _ = speeds.append(1000) %} + {% endif %} + {% endif %} + {% endfor %} + {% else %} + {% if ('module' in default_interface_fact) or (default_interface_fact['type'] == 'bond') %} + {% set speed = (default_interface_fact['speed'] | default(1000)) | string %} + {% if speed == "-1" %} + {% set _ = speeds.append(1000) %} + {% else %} + {% set _ = speeds.append(speed | int) %} + {% endif %} + {% else %} + {% set _ = speeds.append(1000) %} + {% endif %} + {% endif %} + {% set interface_speed = ((speeds | min) * 0.20) | int %} + {{ ((interface_speed | int) > 750) | ternary(750, interface_speed) }} +elasticsearch_data_node_details: >- + {{ + (data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) | list + }} +logstash_data_node_details: >- + {{ + (data_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ logstash_beat_input_port) | list + }} + +# based on the assignment of roles to hosts, set per host booleans +master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}" +data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}" +elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) >= 24) | ternary(24, ansible_processor_cores) }}" + +# Set a data node facts. The data nodes, in the case of elasticsearch are also +# ingest nodes. +elasticsearch_number_of_replicas: "{{ ((data_nodes | length) > 2) | ternary('2', ((data_nodes | length) > 1) | ternary('1', '0')) }}" +elasticsearch_data_hosts: |- + {% set nodes = elasticsearch_data_node_details %} + {% if inventory_hostname in data_nodes %} + {% set _ = nodes.insert(0, '127.0.0.1:' ~ elastic_port) %} + {% endif %} + {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} + {{ data_hosts }} +logstash_data_hosts: |- + {% set nodes = logstash_data_node_details %} + {% if inventory_hostname in data_nodes %} + {% set _ = nodes.insert(0, '127.0.0.1:' ~ logstash_beat_input_port) %} + {% endif %} + {% set data_hosts = nodes | shuffle(seed=inventory_hostname) %} + {{ data_hosts }} diff --git a/elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml b/elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml new file mode 100644 index 00000000..78d93326 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/defaults/main.yml @@ -0,0 +1,22 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Adds option to set the UID/GID of a given service user. +# service_group_gid: 5000 +# service_owner_uid: 5000 + +# Option to define third memory +q_mem: "{{ (ansible_memtotal_mb | int) // 3 }}" + +# Option to define half memory +h_mem: "{{ (ansible_memtotal_mb | int) // 2 }}" diff --git a/elk_metrics_6x/roles/elastic_dependencies/meta/main.yml b/elk_metrics_6x/roles/elastic_dependencies/meta/main.yml new file mode 100644 index 00000000..5d1edce5 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +allow_duplicates: true +galaxy_info: + author: OpenStack + description: Elastic v6.x dependencies role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: [] diff --git a/elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml b/elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml new file mode 100644 index 00000000..36f2db2d --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml @@ -0,0 +1,162 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Check for service_name var + fail: + msg: >- + The required variable [ service_name ] is undefined. + when: + - service_name is undefined + +- name: Check for service_owner var + fail: + msg: >- + The required variable [ service_owner ] is undefined. + when: + - service_owner is undefined + +- name: Check for service_group var + fail: + msg: >- + The required variable [ service_group ] is undefined. + when: + - service_group is undefined + +- name: Load service variables + include_vars: "vars_{{ service_name }}.yml" + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Set elastic heap defaults + set_fact: + elastic_heap_size_default: "{{ _elastic_heap_size_default }}" + +- name: Configure systcl vm.max_map_count=262144 on container hosts + sysctl: + name: "vm.max_map_count" + value: "262144" + state: "present" + reload: "yes" + delegate_to: "{{ physical_host }}" + tags: + - sysctl + +- name: Physical host block + block: + - name: Check for directory + stat: + path: "/var/lib/{{ service_name }}" + register: service_dir + + - name: Check for existing rollup job + debug: + msg: >- + The service data directory [ /var/lib/{{ service_name }} ] already + exists. To ensure no data is lost, the linked directory path to + [ /openstack/{{ inventory_hostname }}/{{ service_name }} ] will not be + created for this host. + when: + - service_dir.stat.isdir is defined and + service_dir.stat.isdir + + - name: Ensure service directories data-path exists + file: + path: "/openstack/{{ inventory_hostname }}/{{ service_name }}" + state: "directory" + owner: "{{ service_owner }}" + group: "{{ service_group }}" + when: + - service_dir.stat.isdir is defined and + not service_dir.stat.isdir + + - name: Ensure data link exists + file: + src: "/openstack/{{ inventory_hostname }}/{{ service_name }}" + dest: "/var/lib/{{ service_name }}" + owner: "{{ service_owner }}" + group: "{{ service_group }}" + state: link + when: + - service_dir.stat.isdir is defined and + not service_dir.stat.isdir + when: + - physical_host == inventory_hostname + +- name: elasticsearch datapath bind mount + lxc_container: + name: "{{ inventory_hostname }}" + container_command: | + [[ ! -d "/var/lib/{{ service_name }}" ]] && mkdir -p "/var/lib/{{ service_name }}" + container_config: + - "lxc.mount.entry=/openstack/{{ inventory_hostname }}/{{ service_name }} var/lib/{{ service_name }} none bind 0 0" + delegate_to: "{{ physical_host }}" + when: + - physical_host != inventory_hostname + - container_tech | default('lxc') == 'lxc' + +- name: Ensure Java is installed + package: + name: "{{ elastic_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + install_recommends: yes + update_cache: yes + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + tags: + - package_install + +- name: Create the system group + group: + name: "{{ service_group }}" + gid: "{{ service_group_gid | default(omit) }}" + state: "present" + system: "yes" + +- name: Create the system user + user: + name: "{{ service_owner }}" + uid: "{{ service_owner_uid | default(omit) }}" + group: "{{ service_group }}" + shell: "/bin/false" + system: "yes" + createhome: "yes" + home: "/var/lib/{{ service_name }}" + +- name: Ensure service directories exists + file: + path: "/etc/{{ service_name }}" + state: "directory" + owner: "{{ service_owner }}" + group: "{{ service_group }}" + +- name: Drop jvm conf file(s) + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - src: templates/jvm.options.j2 + dest: /etc/{{ service_name }}/jvm.options diff --git a/elk_metrics_6x/roles/elastic_dependencies/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_dependencies/vars/ubuntu.yml new file mode 100644 index 00000000..3e016058 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_distro_packages: + - openjdk-8-jre diff --git a/elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml new file mode 100644 index 00000000..0d0180a2 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_elasticsearch.yml @@ -0,0 +1,15 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set memory fact to half +_elastic_heap_size_default: "{{ ((h_mem | int) > 30720) | ternary(30720, h_mem) }}" diff --git a/elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml new file mode 100644 index 00000000..8b34cc76 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_dependencies/vars/vars_logstash.yml @@ -0,0 +1,15 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set quarter memory fact +_elastic_heap_size_default: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}" diff --git a/elk_metrics_6x/roles/elastic_filebeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_filebeat/handlers/main.yml new file mode 100644 index 00000000..5fdc924a --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart filebeat + systemd: + name: "filebeat" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_filebeat/meta/main.yml b/elk_metrics_6x/roles/elastic_filebeat/meta/main.yml new file mode 100644 index 00000000..95bb8f59 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x filebeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml b/elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml new file mode 100644 index 00000000..4b917ed7 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + filebeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml new file mode 100644 index 00000000..0d4a1c2e --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml @@ -0,0 +1,177 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure beat is installed + package: + name: "{{ filebeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart filebeat + tags: + - package_install + +- name: Create filebeat systemd service config dir + file: + path: "/etc/systemd/system/filebeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "/etc/systemd/system/filebeat.service.d/{{ item.dest }}" + mode: "0644" + with_items: + - { src: "systemd.general-overrides.conf.j2", dest: "filebeat-overrides.conf" } + notify: + - Enable and restart filebeat + +- name: Check for apache + stat: + path: /etc/apache2 + register: apache2 + +- name: Check for auditd + stat: + path: /etc/audit + register: audit + +- name: Check for ceph + stat: + path: /var/log/ceph + register: ceph + +- name: Check for cinder + stat: + path: /var/log/cinder + register: cinder + +- name: Check for glance + stat: + path: /var/log/glance + register: glance + +- name: Check for heat + stat: + path: /var/log/heat + register: heat + +- name: Check for horizon + stat: + path: /var/log/horizon + register: horizon + +- name: Check for httpd + stat: + path: /var/log/httpd + register: httpd + +- name: Check for keystone + stat: + path: /var/log/keystone + register: keystone + +- name: Check for mysql + stat: + path: /var/lib/mysql + register: mysql + +- name: Check for neutron + stat: + path: /var/log/neutron + register: neutron + +- name: Check for nginx + stat: + path: /var/log/nginx + register: nginx + +- name: Check for nova + stat: + path: /var/log/nova + register: nova + +- name: Check for octavia + stat: + path: /var/log/octavia + register: octavia + +- name: Check for swift + stat: + path: /var/log/swift + register: swift + +- name: Check for rabbitmq + stat: + path: /var/lib/rabbitmq + register: rabbitmq + +- name: Check for designate + stat: + path: /var/log/designate + register: designate + +- name: Check for osquery + stat: + path: /var/log/osquery/osqueryd.results.log + register: osquery + +- name: Set discovery facts + set_fact: + apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}" + nginx_enabled: "{{ nginx.stat.exists | bool }}" + auditd_enabled: "{{ audit.stat.exists | bool }}" + mysql_enabled: "{{ (mysql.stat.exists | bool) or (inventory_hostname in groups['galera_all'] | default([])) }}" + ceph_enabled: "{{ (ceph.stat.exists | bool) or (inventory_hostname in groups['ceph_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*ceph.*') | list | length) > 0) }}" + cinder_enabled: "{{ (cinder.stat.exists | bool) or (inventory_hostname in groups['cinder_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*cinder.*') | list | length) > 0) }}" + glance_enabled: "{{ (glance.stat.exists | bool) or (inventory_hostname in groups['glance_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*glance.*') | list | length) > 0) }}" + heat_enabled: "{{ (heat.stat.exists | bool) or (inventory_hostname in groups['heat_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*heat.*') | list | length) > 0) }}" + horizon_enabled: "{{ (horizon.stat.exists | bool) or (inventory_hostname in groups['horizon_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*horizon.*') | list | length) > 0) }}" + keystone_enabled: "{{ (keystone.stat.exists | bool) or (inventory_hostname in groups['keystone_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*keystone.*') | list | length) > 0) }}" + neutron_enabled: "{{ (neutron.stat.exists | bool) or (inventory_hostname in groups['neutron_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*neutron.*') | list | length) > 0) }}" + nova_enabled: "{{ (nova.stat.exists | bool) or (inventory_hostname in groups['nova_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*nova.*') | list | length) > 0) }}" + octavia_enabled: "{{ (octavia.stat.exists | bool) or (inventory_hostname in groups['octavia_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*octavia.*') | list | length) > 0) }}" + swift_enabled: "{{ (swift.stat.exists | bool) or (inventory_hostname in groups['swift_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*swift.*') | list | length) > 0) }}" + rabbitmq_enabled: "{{ (rabbitmq.stat.exists | bool) or (inventory_hostname in groups['rabbitmq_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*rabbit.*') | list | length) > 0) }}" + designate_enabled: "{{ (designate.stat.exists | bool) or (inventory_hostname in groups['designate_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*designate.*') | list | length) > 0) }}" + osquery_enabled: "{{ osquery.stat.exists | bool }}" + +- name: Drop Filebeat conf file + template: + src: templates/filebeat.yml.j2 + dest: /etc/filebeat/filebeat.yml + notify: + - Enable and restart filebeat + +- include_tasks: filebeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_filebeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_filebeat/vars/ubuntu.yml new file mode 100644 index 00000000..2f055205 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_filebeat/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filebeat_distro_packages: + - filebeat diff --git a/elk_metrics_6x/roles/elastic_heartbeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_heartbeat/handlers/main.yml new file mode 100644 index 00000000..ac87f12c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart heartbeat + systemd: + name: "heartbeat-elastic" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_heartbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_heartbeat/meta/main.yml new file mode 100644 index 00000000..761d880e --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x heartbeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml b/elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml new file mode 100644 index 00000000..b6d412a1 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + heartbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml new file mode 100644 index 00000000..f52185ba --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml @@ -0,0 +1,63 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure beat is installed + package: + name: "{{ heartbeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart heartbeat + tags: + - package_install + +- name: Create heartbeat systemd service config dir + file: + path: "/etc/systemd/system/heartbeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Create heartbeat configs + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "/etc/systemd/system/heartbeat.service.d/heartbeat-overrides.conf" + - src: templates/heartbeat.yml.j2 + dest: /etc/heartbeat/heartbeat.yml + notify: + - Enable and restart heartbeat + +- include_tasks: heartbeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_heartbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_heartbeat/vars/ubuntu.yml new file mode 100644 index 00000000..4a32d6b7 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_heartbeat/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +heartbeat_distro_packages: + - heartbeat-elastic diff --git a/elk_metrics_6x/roles/elastic_journalbeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_journalbeat/handlers/main.yml new file mode 100644 index 00000000..938ba819 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart journalbeat + systemd: + name: "journalbeat" + enabled: true + state: restarted + daemon_reload: yes + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml new file mode 100644 index 00000000..cc4a5063 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x journalbeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: go_install_1.10 diff --git a/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml b/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml new file mode 100644 index 00000000..473dffa0 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Notice + debug: + msg: >- + The setup task for templates and dashboards is broken due to missing + configuration. Once the following issue + [ https://github.com/mheese/journalbeat/issues/136 ] is + resolved, this should be enabled. + +# - name: Load templates +# shell: >- +# /usr/local/bin/journalbeat -setup +# {{ item }} +# -E 'output.logstash.enabled=false' +# -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' +# -e -v +# with_items: +# - "--template" +# - "--dashboards" +# register: templates +# until: templates is success +# retries: 5 +# delay: 5 +# run_once: true +# tags: +# - setup diff --git a/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml new file mode 100644 index 00000000..24535b2f --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml @@ -0,0 +1,134 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Check for journal directory + stat: + path: /var/log/journal + register: journal_dir + +- name: exit playbook after uninstall + meta: end_play + when: + - not journal_dir.stat.exists | bool + +- name: Ensure beat is installed + package: + name: "{{ journalbeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart journalbeat + tags: + - package_install + +- name: create the system group + group: + name: "journalbeat" + state: "present" + system: "yes" + +- name: Create the journalbeat user + user: + name: "journalbeat" + group: "journalbeat" + comment: "journalbeat user" + shell: "/bin/false" + createhome: "yes" + home: "/usr/share/journalbeat" + +- name: Create journalbeat data path + file: + path: "{{ item }}" + state: directory + owner: "journalbeat" + group: "journalbeat" + mode: "0755" + with_items: + - "/usr/share/journalbeat" + - "/var/lib/journalbeat" + - "/etc/journalbeat" + +- name: Install journalbeat + shell: /opt/go1.10.1/go/bin/go get -v github.com/mheese/journalbeat + args: + creates: /usr/local/bin/journalbeat + environment: + PATH: "{{ ansible_env.PATH }}:/opt/go1.10.1/go/bin" + GOPATH: /usr/local + notify: + - Enable and restart journalbeat + +- name: Create journalbeat systemd service config dir + file: + path: "/etc/systemd/system/journalbeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Drop journalbeat conf files + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "/etc/systemd/system/journalbeat.service.d/journalbeat-overrides.conf" + - src: templates/journalbeat.yml.j2 + dest: /etc/journalbeat/journalbeat.yml + notify: + - Enable and restart journalbeat + +- name: Run the systemd service role + include_role: + name: systemd_service + private: true + vars: + systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" + systemd_service_restart_changed: false + systemd_services: + - service_name: "journalbeat" + execstarts: + - /usr/local/bin/journalbeat + -c /etc/journalbeat/journalbeat.yml + -path.home /usr/share/journalbeat + -path.config /etc/journalbeat + -path.data /var/lib/journalbeat + -path.logs /var/log/beats + config_overrides: + Service: + EnvironmentFile: "-/etc/default/go1.10.1" + Unit: + Documentation: https://github.com/mheese/journalbeat/blob/master/README.md + Wants: network-online.target + After: network-online.target + +- include_tasks: journalbeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml new file mode 100644 index 00000000..bc1ee7bb --- /dev/null +++ b/elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +journalbeat_distro_packages: + - gcc + - git + - libsystemd-dev diff --git a/elk_metrics_6x/roles/elastic_kibana/defaults/main.yml b/elk_metrics_6x/roles/elastic_kibana/defaults/main.yml new file mode 100644 index 00000000..a197cac3 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kibana_enable_basic_auth: false diff --git a/elk_metrics_6x/roles/elastic_kibana/handlers/main.yml b/elk_metrics_6x/roles/elastic_kibana/handlers/main.yml new file mode 100644 index 00000000..12f4111f --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/handlers/main.yml @@ -0,0 +1,37 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart nginx + systemd: + name: "nginx" + enabled: true + state: restarted + when: + - (elk_package_state | default('present')) != 'absent' + listen: Enable and restart services + tags: + - config + +- name: Enable and restart kibana + systemd: + name: "kibana" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + listen: Enable and restart services + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_kibana/meta/main.yml b/elk_metrics_6x/roles/elastic_kibana/meta/main.yml new file mode 100644 index 00000000..3214fdf3 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x kibana role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_kibana/tasks/main.yml b/elk_metrics_6x/roles/elastic_kibana/tasks/main.yml new file mode 100644 index 00000000..9391e964 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/tasks/main.yml @@ -0,0 +1,84 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure distro packages are installed + package: + name: "{{ kibana_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: yes + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart services + tags: + - package_install + +- name: create kibana user to access web interface + htpasswd: + path: "/etc/nginx/htpasswd.users" + name: "{{ kibana_username }}" + password: "{{ kibana_password }}" + owner: root + mode: 0644 + when: + - kibana_enable_basic_auth + +- name: Drop Nginx default conf file + template: + src: templates/nginx_default.j2 + dest: /etc/nginx/sites-available/default + notify: + - Enable and restart services + +- name: Create kibana systemd service config dir + file: + path: "/etc/systemd/system/kibana.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "/etc/systemd/system/kibana.service.d/{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "kibana-overrides.conf" + notify: + - Enable and restart services + +- name: Drop kibana conf file + template: + src: templates/kibana.yml.j2 + dest: /etc/kibana/kibana.yml + mode: "0666" + notify: + - Enable and restart services diff --git a/elk_metrics_6x/roles/elastic_kibana/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_kibana/vars/ubuntu.yml new file mode 100644 index 00000000..780eb881 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_kibana/vars/ubuntu.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kibana_distro_packages: + - apache2-utils + - kibana + - nginx + - python-passlib diff --git a/elk_metrics_6x/roles/elastic_logstash/defaults/main.yml b/elk_metrics_6x/roles/elastic_logstash/defaults/main.yml new file mode 100644 index 00000000..2c9a92d2 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +temp_dir: /var/lib/logstash/tmp +logstash_pipelines: "{{lookup('template', 'templates/logstash-pipelines.yml.j2') }}" + +# Set processor cores fact +q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_core | int) * 2 }}" + +# Set logstash facts +logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}" +elastic_log_rotate_path: "/var/log/logstash" diff --git a/elk_metrics_6x/roles/elastic_logstash/handlers/main.yml b/elk_metrics_6x/roles/elastic_logstash/handlers/main.yml new file mode 100644 index 00000000..8ea50751 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart logstash + systemd: + name: "logstash" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_logstash/meta/main.yml b/elk_metrics_6x/roles/elastic_logstash/meta/main.yml new file mode 100644 index 00000000..f5684dc8 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/meta/main.yml @@ -0,0 +1,37 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x logstash role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories + - role: elastic_dependencies + service_name: logstash + service_owner: logstash + service_group: logstash diff --git a/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_arcsight.yml b/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_arcsight.yml new file mode 100644 index 00000000..f9588ca8 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_arcsight.yml @@ -0,0 +1,62 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Initialise arcsight local facts + ini_file: + dest: "/etc/ansible/facts.d/logstash.fact" + section: arcsight + option: "initialise" + value: true + +- name: refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + tags: + - nova-config + +- name: Setup arcsight smart connector + shell: >- + /usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }} + -M "arcsight.var.input.smartconnector.bootstrap_servers={{ item.host }}:{{ item.port }}" + -M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}" + -M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}" + with_items: "{{ logstash_arcsight_smart_connectors }}" + run_once: true + register: smart_connector + until: smart_connector is success + retries: 5 + delay: 5 + +- name: Setup arcsight event broker + shell: >- + /usr/local/bin/logstash --modules arcsight {{ (ansible_local['arcsight'][item.host] is defined) | ternary('', '--setup') }} + -M "arcsight.var.input.eventbroker.bootstrap_servers={{ item.host }}:{{ item.port }}" + -M "arcsight.var.elasticsearch.hosts=localhost:{{ elastic_port }}" + -M "arcsight.var.kibana.host={{ hostvars[groups['kibana'][0]]['ansible_host'] }}:{{ kibana_port }}" + with_items: "{{ logstash_arcsight_event_brokers }}" + run_once: true + register: event_broker + until: event_broker is success + retries: 5 + delay: 5 + +- name: Set arcsight local facts + ini_file: + dest: "/etc/ansible/facts.d/logstash.fact" + section: arcsight + option: "{{ item.host }}" + value: "{{ item.port }}" + with_items: "{{ logstash_arcsight_smart_connectors | union(logstash_arcsight_event_brokers) }}" diff --git a/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_kafka_ssl.yml b/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_kafka_ssl.yml new file mode 100644 index 00000000..fdcad549 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/tasks/logstash_kafka_ssl.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Copy kafka keystore into place + copy: + src: "{{ logstash_kafka_ssl_keystore_location }}" + dest: "/var/lib/logstash/{{ logstash_kafka_ssl_keystore_location | basename }}" + +- name: Copy kafka truststore into place + copy: + src: "{{ logstash_kafka_ssl_truststore_location }}" + dest: "/var/lib/logstash/{{ logstash_kafka_ssl_truststore_location | basename }}" diff --git a/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml b/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml new file mode 100644 index 00000000..dec0dab0 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/tasks/main.yml @@ -0,0 +1,129 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure Logstash is installed + package: + name: "{{ logstash_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: yes + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart logstash + tags: + - package_install + +- name: Create logstash systemd service config dir + file: + path: "/etc/systemd/system/logstash.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "/etc/systemd/system/logstash.service.d/{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "logstash-overrides.conf" + notify: + - Enable and restart logstash + +- name: Create patterns directory + file: + name: "/opt/logstash/patterns" + owner: "logstash" + group: "logstash" + state: directory + tags: + - logstash-patterns + +- name: Logstash Extra Patterns + template: + src: "extras" + dest: "/opt/logstash/patterns/extras" + owner: "logstash" + group: "logstash" + when: + - logstash_deploy_filters + notify: + - Enable and restart logstash + tags: + - logstash-filters + - config + +- name: Run kafka ssl deployment + include_tasks: logstash_kafka_ssl.yml + when: + - logstash_kafka_options is defined + - logstash_kafka_ssl_keystore_location is defined + +- name: Drop logstash conf file(s) + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - src: templates/logstash.yml.j2 + dest: /etc/logstash/logstash.yml + - src: templates/logstash-pipelines.yml.j2 + dest: /etc/logstash/pipelines.yml + - src: "templates/logrotate.j2" + dest: "/etc/logrotate.d/logstash" + notify: + - Enable and restart logstash + tags: + - config + +- name: Ensure logstash ownership + file: + path: "/var/lib/logstash/" + owner: logstash + group: logstash + recurse: true + register: l_perms + until: l_perms is success + retries: 3 + delay: 1 + +- name: Ensure logstash tmp dir + file: + path: "/var/lib/logstash/tmp" + state: directory + owner: "logstash" + group: "logstash" + mode: "0750" + +- name: Deploy arcsight collector + include_tasks: logstash_arcsight.yml + when: + - logstash_arcsight_smart_connectors or + logstash_arcsight_event_brokers diff --git a/elk_metrics_6x/roles/elastic_logstash/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_logstash/vars/ubuntu.yml new file mode 100644 index 00000000..716310e6 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_logstash/vars/ubuntu.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logstash_distro_packages: + - logrotate + - logstash diff --git a/elk_metrics_6x/roles/elastic_metricbeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_metricbeat/handlers/main.yml new file mode 100644 index 00000000..9493df01 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart metricbeat + systemd: + name: "metricbeat" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_metricbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_metricbeat/meta/main.yml new file mode 100644 index 00000000..73585980 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x metricbeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml new file mode 100644 index 00000000..2229e242 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml @@ -0,0 +1,236 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure beat is installed + package: + name: "{{ metricbeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart metricbeat + tags: + - package_install + +- name: Check for apache + stat: + path: /etc/apache2/sites-available + register: apache2 + +- name: Check for ceph + stat: + path: /etc/ceph + register: ceph + +# gather ceph stats from localhost +# except when a list of mons is provided +- name: Set ceph stats hosts + set_fact: + ceph_stats_hosts: |- + {% set ceph_stats = [] %} + {% if (ceph_mons is defined) and (ceph_mons | length > 0) %} + {% for mon in ceph_mons %} + {% set _ = ceph_stats.insert(loop.index, (mon + ":5000")) %} + {% endfor %} + {% else %} + {% set ceph_stats = [ ansible_hostname + ":5000" ] %} + {% endif %} + {{ ceph_stats }} + +- name: Check for etcd + stat: + path: /etc/etcd + register: etcd + +- name: Check for docker + stat: + path: /var/run/docker.sock + register: docker + +- name: Check for haproxy + stat: + path: /etc/haproxy + register: haproxy + +- name: Check for httpd + stat: + path: /etc/httpd + register: httpd + +- name: Check for kvm + stat: + path: /var/run/libvirt/libvirt-sock + register: kvm + +- name: Check for memcached + stat: + path: /etc/memcached.conf + register: memcached + +- name: Check for mysql + stat: + path: /var/lib/mysql + register: mysql + +- name: Check for nginx + stat: + path: /etc/nginx/nginx.conf + register: nginx + +- name: Check for rabbitmq + stat: + path: /var/lib/rabbitmq + register: rabbitmq + +- name: Check for uwsgi + stat: + path: /etc/uwsgi + register: uwsgi + +- name: Check for uwsgi stats sockets + find: + paths: /tmp + file_type: any + patterns: '*uwsgi-stats.sock' + register: uwsgi_find_sockets + +- name: Set discovery facts + set_fact: + apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}" + + # enable ceph on: cinder volume hosts when we have a list of ceph mons + # otherwise: all hosts which have /etc/ceph + ceph_enabled: |- + {% set ceph_detect = false %} + {% if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups['cinder_volume']) %} + {% set ceph_detect = true %} + {% else %} + {% set ceph_detect = ceph.stat.exists | bool %} + {% endif %} + {{ ceph_detect }} + + docker_enabled: "{{ docker.stat.exists | bool }}" + etcd_enabled: "{{ etcd.stat.exists | bool }}" + haproxy_enabled: "{{ haproxy.stat.exists | bool }}" + kvm_enabled: "{{ kvm.stat.exists | bool }}" + memcached_enabled: "{{ memcached.stat.exists | bool }}" + mysql_enabled: "{{ mysql.stat.exists | bool }}" + nginx_enabled: "{{ nginx.stat.exists | bool }}" + rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}" + uwsgi_enabled: "{{ uwsgi.stat.exists | bool }}" + uwsgi_sockets: "{{ uwsgi_find_sockets }}" + +# Apache 2 stats enablement +- name: Drop apache2 stats site config + template: + src: apache-status.conf.j2 + dest: /etc/apache2/sites-available/apache-status.conf + when: apache_enabled + +- name: Enable apache2 stats site + file: + src: /etc/apache2/sites-available/apache-status.conf + dest: /etc/apache2/sites-enabled/apache-status.conf + state: link + when: apache_enabled + +- name: Ensure apache2 stats mode is enabled + apache2_module: + name: status + state: present + register: apache_status_mod + when: apache_enabled + +- name: Reload apache2 + service: + name: apache2 + state: reloaded + when: + - apache_enabled + - apache_status_mod is changed + +# NGINX stats enablement +- name: Drop nginx stats site config + template: + src: nginx-status.conf.j2 + dest: /etc/nginx/sites-available/nginx-status.conf + when: nginx_enabled + +- name: Enable nginx stats site + file: + src: /etc/nginx/sites-available/nginx-status.conf + dest: /etc/nginx/sites-enabled/nginx-status.conf + state: link + register: nginx_status + when: nginx_enabled + +- name: Reload nginx + service: + name: nginx + state: reloaded + when: + - nginx_enabled + - nginx_status is changed + +- name: Create metricbeat systemd service config dir + file: + path: "/etc/systemd/system/metricbeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + when: + - ansible_service_mgr == 'systemd' + +- name: Apply systemd options + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + when: + - ansible_service_mgr == 'systemd' + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "/etc/systemd/system/metricbeat.service.d/metricbeat-overrides.conf" + notify: + - Enable and restart metricbeat + +- name: Drop metricbeat conf file + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + with_items: + - src: templates/metricbeat.yml.j2 + dest: /etc/metricbeat/metricbeat.yml + notify: + - Enable and restart metricbeat + +- include_tasks: metricbeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml b/elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml new file mode 100644 index 00000000..86d56690 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + metricbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_metricbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_metricbeat/vars/ubuntu.yml new file mode 100644 index 00000000..2eecca08 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_metricbeat/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +metricbeat_distro_packages: + - metricbeat diff --git a/elk_metrics_6x/roles/elastic_packetbeat/handlers/main.yml b/elk_metrics_6x/roles/elastic_packetbeat/handlers/main.yml new file mode 100644 index 00000000..068f373c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart packetbeat + systemd: + name: "packetbeat" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elastic_packetbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_packetbeat/meta/main.yml new file mode 100644 index 00000000..c403c434 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x packetbeat role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml new file mode 100644 index 00000000..d634ea48 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml @@ -0,0 +1,63 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Ensure beat is installed + package: + name: "{{ packetbeat_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: true + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart packetbeat + tags: + - package_install + +- name: Create packetbeat systemd service config dir + file: + path: "/etc/systemd/system/packetbeat.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Drop packetbeat conf files + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + with_items: + - src: "systemd.general-overrides.conf.j2" + dest: "/etc/systemd/system/packetbeat.service.d/packetbeat-overrides.conf" + - src: templates/packetbeat.yml.j2 + dest: /etc/packetbeat/packetbeat.yml + notify: + - Enable and restart packetbeat + +- include_tasks: packetbeat_setup.yml diff --git a/elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml b/elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml new file mode 100644 index 00000000..33dc703d --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Load templates + shell: >- + packetbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -e -v + with_items: + - "--template" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_packetbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_packetbeat/vars/ubuntu.yml new file mode 100644 index 00000000..e949478a --- /dev/null +++ b/elk_metrics_6x/roles/elastic_packetbeat/vars/ubuntu.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +packetbeat_distro_packages: + - tcpdump + - packetbeat diff --git a/elk_metrics_6x/roles/elastic_repositories/meta/main.yml b/elk_metrics_6x/roles/elastic_repositories/meta/main.yml new file mode 100644 index 00000000..c3a7f802 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_repositories/meta/main.yml @@ -0,0 +1,31 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x repositories role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: [] diff --git a/elk_metrics_6x/common_task_install_elk_repo.yml b/elk_metrics_6x/roles/elastic_repositories/tasks/elastic_apt_repos.yml similarity index 84% rename from elk_metrics_6x/common_task_install_elk_repo.yml rename to elk_metrics_6x/roles/elastic_repositories/tasks/elastic_apt_repos.yml index 774e5ae6..9bbd6389 100644 --- a/elk_metrics_6x/common_task_install_elk_repo.yml +++ b/elk_metrics_6x/roles/elastic_repositories/tasks/elastic_apt_repos.yml @@ -24,18 +24,6 @@ tags: - package_install -- name: enable apt-transport-https - apt: - name: apt-transport-https - state: present - update_cache: yes - register: _apt_task - until: _apt_task is success - retries: 3 - delay: 2 - tags: - - package_install - - name: add elk repo to apt sources list apt_repository: repo: "{{ elk_repo.repo }}" diff --git a/elk_metrics_6x/roles/elastic_repositories/tasks/main.yml b/elk_metrics_6x/roles/elastic_repositories/tasks/main.yml new file mode 100644 index 00000000..0708ca46 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_repositories/tasks/main.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Install repo disto packages + package: + name: "{{ elastic_repo_distro_packages }}" + state: present + update_cache: yes + register: _apt_task + until: _apt_task is success + retries: 3 + delay: 2 + tags: + - package_install + +- include_tasks: "elastic_{{ ansible_pkg_mgr }}_repos.yml" diff --git a/elk_metrics_6x/roles/elastic_repositories/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_repositories/vars/ubuntu.yml new file mode 100644 index 00000000..8752df14 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_repositories/vars/ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_repo_distro_packages: + - apt-transport-https diff --git a/elk_metrics_6x/roles/elastic_retention/defaults/main.yml b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml new file mode 100644 index 00000000..a3ae847f --- /dev/null +++ b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_index_retention_algorithm: default diff --git a/elk_metrics_6x/roles/elastic_retention/meta/main.yml b/elk_metrics_6x/roles/elastic_retention/meta/main.yml new file mode 100644 index 00000000..f360643a --- /dev/null +++ b/elk_metrics_6x/roles/elastic_retention/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x retention role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts diff --git a/elk_metrics_6x/roles/elastic_retention/tasks/main.yml b/elk_metrics_6x/roles/elastic_retention/tasks/main.yml new file mode 100644 index 00000000..3b7a52d3 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_retention/tasks/main.yml @@ -0,0 +1,35 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Query es storage + uri: + url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs" + method: GET + register: elk_data + until: elk_data is success + retries: 5 + delay: 5 + run_once: true + +- name: Load data node variables + include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml" + tags: + - always + +- name: Set retention facts + set_fact: "elastic_{{ item.key }}_retention={{ (es_assumed_usable_storage_per_node | int) // ((item.value | int) * (es_storage_multiplier | int)) }}" + when: + - hostvars[inventory_hostname]["elastic_" + item.key + "_retention"] is undefined + with_dict: "{{ elastic_beat_retention_policy_hosts }}" diff --git a/elk_metrics_6x/roles/elastic_retention/vars/calculate_index_retention_default.yml b/elk_metrics_6x/roles/elastic_retention/vars/calculate_index_retention_default.yml new file mode 100644 index 00000000..4a060a8a --- /dev/null +++ b/elk_metrics_6x/roles/elastic_retention/vars/calculate_index_retention_default.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set available storage fact +es_total_available_storage: "{{ ((elk_data['json']['nodes'].values() | list) | map(attribute='fs.total.total_in_bytes') | list | sum) // 1024 // 1024 }}" + +# Set assumed buffer storage fact +es_assumed_buffer_storage: "{{ ((es_total_available_storage | int) * 0.25) | round | int }}" + +# Set usable buffer storage fact(s) +es_usable_buffer_storage: "{{ (es_total_available_storage | int) - (es_assumed_buffer_storage | int) }}" +es_expected_storage: "{{ ((elastic_beat_retention_policy_hosts.values() | map('int') | list) | sum) * (elastic_beat_storage_constant | int) }}" + +# Set buffer storage fact +es_assumed_usable_storage_per_node: "{{ (es_usable_buffer_storage | int) // (data_nodes | length | int) }}" + +# Set storage the mulitplier +es_storage_multiplier: "{{ ((es_usable_buffer_storage | int) < (es_expected_storage | int)) | ternary(((elastic_beat_storage_constant | int) * 2), elastic_beat_storage_constant | int) }}" diff --git a/elk_metrics_6x/roles/elastic_rollup/defaults/main.yml b/elk_metrics_6x/roles/elastic_rollup/defaults/main.yml new file mode 100644 index 00000000..f72162fd --- /dev/null +++ b/elk_metrics_6x/roles/elastic_rollup/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_allow_rollup_purge: false diff --git a/elk_metrics_6x/roles/elastic_rollup/meta/main.yml b/elk_metrics_6x/roles/elastic_rollup/meta/main.yml new file mode 100644 index 00000000..2a2aed2c --- /dev/null +++ b/elk_metrics_6x/roles/elastic_rollup/meta/main.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x rollup role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_retention diff --git a/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml b/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml new file mode 100644 index 00000000..0ce194d9 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml @@ -0,0 +1,96 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Get rollup job + uri: + url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}" + method: GET + register: check_rollup + until: check_rollup is success + retries: 3 + delay: 5 + run_once: true + +- name: Check for existing rollup job + debug: + msg: >- + An existing rollup job was found for {{ index_name }}. In order to + re-create this rollup job the old job will need to be purged. If you're + OK with the old rollup job being purged, add the following option + `elastic_allow_rollup_purge=yes` to the command line and rerun the + playbook. + when: + - check_rollup['json']['jobs'] | length > 0 + - not elastic_allow_rollup_purge | bool + +- name: Create rollup block + block: + - name: Set retention days fact + set_fact: + days_until_rollup: |- + {% set index_retention = [] %} + {% for item in ansible_play_hosts %} + {% set _ = index_retention.append((hostvars[item]['elastic_' + index_name + '_retention'] | int) // 3) %} + {% endfor %} + {{ index_retention | min }} + run_once: true + + - name: Create rollup job + uri: + url: "{{ item.url }}" + method: "{{ item.method }}" + body: "{{ item.index_options | to_json }}" + status_code: "{{ item.status_code }}" + body_format: json + register: elk_indexes + until: elk_indexes is success + retries: 3 + delay: 5 + when: + - hostvars[inventory_hostname]['elastic_' + index_name + '_retention'] > days_until_rollup + with_items: + - url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}/_stop" + method: POST + status_code: 200,404 + index_options: {} + - url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}" + method: DELETE + status_code: 200,404 + index_options: {} + - url: "http://{{ coordination_nodes[0] }}/rollup_{{ index_name }}" + method: DELETE + status_code: 200,404 + index_options: {} + - url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}" + method: PUT + status_code: 200 + index_options: + index_pattern: "{{ index_name }}-*" + rollup_index: "rollup_{{ index_name }}" + cron: "*/30 * * * * ?" + page_size: 1000 + groups: + date_histogram: + field: "@timestamp" + interval: "1h" + delay: "{{ days_until_rollup }}d" + - url: "http://{{ coordination_nodes[0] }}/_xpack/rollup/job/rollup_{{ index_name }}/_start" + method: POST + status_code: 200 + index_options: {} + run_once: true + when: + - check_rollup['json']['jobs'] | length < 1 or + elastic_allow_rollup_purge | bool diff --git a/elk_metrics_6x/roles/elasticsearch/defaults/main.yml b/elk_metrics_6x/roles/elasticsearch/defaults/main.yml new file mode 100644 index 00000000..aedbb705 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/defaults/main.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elastic_log_rotate_path: "/var/log/elasticsearch" + +temp_dir: /var/lib/elasticsearch/tmp +nfs_query: "[?fstype=='nfs' || fstype=='nfs4']" diff --git a/elk_metrics_6x/roles/elasticsearch/handlers/main.yml b/elk_metrics_6x/roles/elasticsearch/handlers/main.yml new file mode 100644 index 00000000..f1403d58 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/handlers/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Enable and restart elastic + systemd: + name: "elasticsearch" + enabled: true + state: restarted + daemon_reload: true + when: + - (elk_package_state | default('present')) != 'absent' + tags: + - config diff --git a/elk_metrics_6x/roles/elasticsearch/meta/main.yml b/elk_metrics_6x/roles/elasticsearch/meta/main.yml new file mode 100644 index 00000000..6010c208 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/meta/main.yml @@ -0,0 +1,37 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x elasticsearch role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: + - role: elastic_data_hosts + - role: elastic_repositories + - role: elastic_dependencies + service_name: elasticsearch + service_owner: elasticsearch + service_group: elasticsearch diff --git a/elk_metrics_6x/roles/elasticsearch/tasks/elasticsearch_nfs_setup.yml b/elk_metrics_6x/roles/elasticsearch/tasks/elasticsearch_nfs_setup.yml new file mode 100644 index 00000000..2234f482 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/tasks/elasticsearch_nfs_setup.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Ensure nfs client is available if necessary + package: + name: nfs-common + state: present + +- name: Ensure backup filesystems are mounted + mount: + fstype: "{{ item.fstype }}" + src: "{{ item.src }}" + opts: "{{ item.opts }}" + path: "{{ item.path }}" + state: "{{ item.state }}" + with_items: + - "{{ elastic_shared_fs_repos }}" diff --git a/elk_metrics_6x/roles/elasticsearch/tasks/main.yml b/elk_metrics_6x/roles/elasticsearch/tasks/main.yml new file mode 100644 index 00000000..b650f9ec --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/tasks/main.yml @@ -0,0 +1,102 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather variables for each operating system + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml" + - "{{ ansible_os_family | lower }}.yml" + tags: + - always + +- name: Set elasticsearch variables + include_vars: "vars_{{ ((inventory_hostname in (groups['kibana'] | default([])) and not inventory_hostname in (groups['elastic-logstash']) | default([]))) | ternary('kibana', 'default') }}.yml" + tags: + - always + +- name: Ensure elasticsearch is installed + package: + name: "{{ elasticsearch_distro_packages }}" + state: "{{ elk_package_state | default('present') }}" + update_cache: yes + register: _package_task + until: _package_task is success + retries: 3 + delay: 2 + notify: + - Enable and restart elastic + tags: + - package_install + +- name: Create elasticsearch systemd service config dir + file: + path: "/etc/systemd/system/elasticsearch.service.d" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Drop elasticsearch conf file + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - src: templates/elasticsearch.yml.j2 + dest: /etc/elasticsearch/elasticsearch.yml + - src: templates/jvm.options.j2 + dest: /etc/elasticsearch/jvm.options + - src: templates/es-log4j2.properties.j2 + dest: /etc/elasticsearch/log4j2.properties + - src: "systemd.elasticsearch-overrides.conf.j2" + dest: "/etc/systemd/system/elasticsearch.service.d/elasticsearch-overrides.conf" + notify: + - Enable and restart elastic + tags: + - config + +- name: Ensure elasticsearch ownership + file: + path: "/var/lib/elasticsearch/" + owner: elasticsearch + group: elasticsearch + recurse: true + register: e_perms + until: e_perms is success + retries: 3 + delay: 1 + tags: + - config + +- name: Ensure elasticsearch tmp dir + file: + path: "/var/lib/elasticsearch/tmp" + state: directory + owner: "elasticsearch" + group: "elasticsearch" + mode: "0750" + +- name: Create logrotate config + template: + src: "templates/logrotate.j2" + dest: "/etc/logrotate.d/elasticsearch" + +- include_tasks: "elasticsearch_nfs_setup.yml" + when: + - elastic_shared_fs_repos is defined + - (elastic_shared_fs_repos | json_query(nfs_query)) | length > 0 diff --git a/elk_metrics_6x/roles/elasticsearch/vars/ubuntu.yml b/elk_metrics_6x/roles/elasticsearch/vars/ubuntu.yml new file mode 100644 index 00000000..a5948d12 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/vars/ubuntu.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +elasticsearch_distro_packages: + - logrotate + - elasticsearch diff --git a/elk_metrics_6x/roles/elasticsearch/vars/vars_default.yml b/elk_metrics_6x/roles/elasticsearch/vars/vars_default.yml new file mode 100644 index 00000000..147f27f2 --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/vars/vars_default.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set elasticsearch facts +elastic_heap_size: "{{ elastic_heap_size_default }}" diff --git a/elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml b/elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml new file mode 100644 index 00000000..65056b9b --- /dev/null +++ b/elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set kibana as elasticsearch coordinators +elasticsearch_node_master: false +elasticsearch_node_data: false +elasticsearch_node_ingest: false +elastic_coordination_node: true +elastic_heap_size: "{{ (elastic_heap_size_default | int) // 3 }}" +elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 4) | ternary(4, (ansible_processor_cores // 2)) }}" diff --git a/elk_metrics_6x/roles/go_install_1.10/handlers/main.yml b/elk_metrics_6x/roles/go_install_1.10/handlers/main.yml new file mode 100644 index 00000000..4bc61095 --- /dev/null +++ b/elk_metrics_6x/roles/go_install_1.10/handlers/main.yml @@ -0,0 +1,14 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/elk_metrics_6x/roles/go_install_1.10/meta/main.yml b/elk_metrics_6x/roles/go_install_1.10/meta/main.yml new file mode 100644 index 00000000..3998985c --- /dev/null +++ b/elk_metrics_6x/roles/go_install_1.10/meta/main.yml @@ -0,0 +1,31 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x go install role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - xenial + categories: + - cloud + - development + - elasticsearch + - elastic-stack +dependencies: [] diff --git a/elk_metrics_6x/roles/go_install_1.10/tasks/go_install.yml b/elk_metrics_6x/roles/go_install_1.10/tasks/go_install.yml new file mode 100644 index 00000000..904bb619 --- /dev/null +++ b/elk_metrics_6x/roles/go_install_1.10/tasks/go_install.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: GET go + get_url: + url: "https://dl.google.com/go/go1.10.1.linux-amd64.tar.gz" + dest: "/opt/go1.10.1.linux-amd64.tar.gz" + checksum: "sha256:72d820dec546752e5a8303b33b009079c15c2390ce76d67cf514991646c6127b" + register: _get_task + until: _get_task is success + retries: 3 + delay: 2 + tags: + - package_install + +- name: Create go directory + file: + path: "/opt/go1.10.1" + state: directory + +- name: Unarchive go + unarchive: + src: "/opt/go1.10.1.linux-amd64.tar.gz" + dest: "/opt/go1.10.1" + remote_src: yes + +- name: Create go defaults file + copy: + content: | + GOROOT=/opt/go1.10.1/go + GOPATH=/usr/local + PATH=${PATH}:${GOROOT}/bin + dest: /etc/default/go1.10.1 diff --git a/elk_metrics_6x/roles/go_install_1.10/tasks/main.yml b/elk_metrics_6x/roles/go_install_1.10/tasks/main.yml new file mode 100644 index 00000000..c7940ca1 --- /dev/null +++ b/elk_metrics_6x/roles/go_install_1.10/tasks/main.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Check for go + stat: + path: /opt/go1.10.1/go/bin/go + register: go_path + +- include_tasks: go_install.yml + when: + - not go_path.stat.exists | bool diff --git a/elk_metrics_6x/setupAPMserver.yml b/elk_metrics_6x/setupAPMserver.yml deleted file mode 100644 index ba3f76f1..00000000 --- a/elk_metrics_6x/setupAPMserver.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load apm-server Dashboards - hosts: apm-server[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - apm-server setup - {{ item }} - -E 'apm-server.host=localhost:8200' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupAuditbeat.yml b/elk_metrics_6x/setupAuditbeat.yml deleted file mode 100644 index e5e100cc..00000000 --- a/elk_metrics_6x/setupAuditbeat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load Auditbeat Dashboards - hosts: hosts[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - auditbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupFilebeat.yml b/elk_metrics_6x/setupFilebeat.yml deleted file mode 100644 index ec67d7e0..00000000 --- a/elk_metrics_6x/setupFilebeat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load Filebeat Dashboards - hosts: hosts[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - filebeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupHeartbeat.yml b/elk_metrics_6x/setupHeartbeat.yml deleted file mode 100644 index 9d648867..00000000 --- a/elk_metrics_6x/setupHeartbeat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load Heartbeat Dashboards - hosts: kibana[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - heartbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupJournalbeat.yml b/elk_metrics_6x/setupJournalbeat.yml deleted file mode 100644 index ecf84f17..00000000 --- a/elk_metrics_6x/setupJournalbeat.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Load Journalbeat Dashboards - hosts: hosts[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - # NOTE(cloudnull): This task is broken at this point due to missing - # configuration. Once the following issue - # [ https://github.com/mheese/journalbeat/issues/136 ] is - # resolved, this should be removed. - - name: exit playbook - meta: end_play - - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - /usr/local/bin/journalbeat -setup - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupKibanaDashboard.yml b/elk_metrics_6x/setupKibanaDashboard.yml index 47d6c3f6..bc9fabeb 100644 --- a/elk_metrics_6x/setupKibanaDashboard.yml +++ b/elk_metrics_6x/setupKibanaDashboard.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Install Kibana Dashboards hosts: "elastic-logstash" become: true diff --git a/elk_metrics_6x/setupMetricbeat.yml b/elk_metrics_6x/setupMetricbeat.yml deleted file mode 100644 index 302b9321..00000000 --- a/elk_metrics_6x/setupMetricbeat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load Metricsbeat Dashboards - hosts: all[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - metricbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/setupPacketbeat.yml b/elk_metrics_6x/setupPacketbeat.yml deleted file mode 100644 index 8312a34e..00000000 --- a/elk_metrics_6x/setupPacketbeat.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Load Packetbeat Dashboards - hosts: hosts[0] - gather_facts: true - vars_files: - - vars/variables.yml - - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml - tags: - - always - - tasks: - - name: Load templates - shell: >- - packetbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -e -v - with_items: - - "--template" - - "--dashboards" - register: templates - until: templates is success - retries: 5 - delay: 5 - - tags: - - beat-setup diff --git a/elk_metrics_6x/showElasticCluster.yml b/elk_metrics_6x/showElasticCluster.yml index 0ecaa29e..5bf38ca6 100644 --- a/elk_metrics_6x/showElasticCluster.yml +++ b/elk_metrics_6x/showElasticCluster.yml @@ -1,4 +1,16 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + - name: Dump cluster node info hosts: elastic-logstash:kibana become: true @@ -8,11 +20,11 @@ environment: "{{ deployment_environment_variables | default({}) }}" - pre_tasks: - - include_tasks: common_task_data_node_hosts.yml + roles: + - role: elastic_data_hosts # display overall cluster setup - tasks: + post_tasks: - block: - name: "Report number of storage nodes" debug: diff --git a/elk_metrics_6x/site-beats-community.yml b/elk_metrics_6x/site-beats-community.yml index 6eb444ff..86891276 100644 --- a/elk_metrics_6x/site-beats-community.yml +++ b/elk_metrics_6x/site-beats-community.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/elk_metrics_6x/site-beats-core.yml b/elk_metrics_6x/site-beats-core.yml index b506abd5..266ab657 100644 --- a/elk_metrics_6x/site-beats-core.yml +++ b/elk_metrics_6x/site-beats-core.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/elk_metrics_6x/site-elka.yml b/elk_metrics_6x/site-elka.yml index 7052a204..bc02fce9 100644 --- a/elk_metrics_6x/site-elka.yml +++ b/elk_metrics_6x/site-elka.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/elk_metrics_6x/site.yml b/elk_metrics_6x/site.yml index f820a587..01b23aa1 100644 --- a/elk_metrics_6x/site.yml +++ b/elk_metrics_6x/site.yml @@ -1,6 +1,4 @@ --- -# Copyright 2018, Rackspace US, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/elk_metrics_6x/templates/jvm.options.j2 b/elk_metrics_6x/templates/jvm.options.j2 index 712699c7..c0130f32 100644 --- a/elk_metrics_6x/templates/jvm.options.j2 +++ b/elk_metrics_6x/templates/jvm.options.j2 @@ -1,8 +1,8 @@ ## JVM configuration {% if (not (elasticsearch_node_master | default(master_node)) | bool) and (not (elasticsearch_node_data | default(data_node)) | bool) %} -{% set heap_size = (elastic_heap_size | int) // 2 %} +{% set heap_size = (elastic_heap_size_default | int) // 2 %} {% else %} -{% set heap_size = (elastic_heap_size | int) %} +{% set heap_size = (elastic_heap_size_default | int) %} {% endif %} # Xms represents the initial size of total heap space -Xms{{ heap_size }}m diff --git a/elk_metrics_6x/tests/testLayout.yml b/elk_metrics_6x/tests/testLayout.yml index 629381c9..418a3457 100644 --- a/elk_metrics_6x/tests/testLayout.yml +++ b/elk_metrics_6x/tests/testLayout.yml @@ -1,4 +1,15 @@ --- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. - import_playbook: run-setup.yml