From c6493a812b19f4a504855357530cde41cd47a267 Mon Sep 17 00:00:00 2001 From: cloudnull Date: Sun, 17 Feb 2019 23:05:15 -0600 Subject: [PATCH] Add monitorstack data collection into ES The monitorstack data collection can export data into elasticsearch. A playbook has been added to deploy the data collection probes which will leverage systemd-timers to run the probes on regular intervals. The systemd timers will be deployed per-probe and run within the utility, compute, and memcached hosts. Any place the probes are deployed an isolated user will ensure to fence the probes from the cluster and limit access. OpenStack probes will only be deployed when an openstack-sdk clouds config is found within the system. Change-Id: Ic5cd5fd51a7e0763c0a2db40af4150b8851bc748 Signed-off-by: cloudnull --- elk_metrics_6x/installMonitorStack.yml | 277 ++++++++++++++++++ .../roles/elastic_retention/defaults/main.yml | 4 + elk_metrics_6x/site.yml | 1 + elk_metrics_6x/vars/variables.yml | 15 + 4 files changed, 297 insertions(+) create mode 100644 elk_metrics_6x/installMonitorStack.yml diff --git a/elk_metrics_6x/installMonitorStack.yml b/elk_metrics_6x/installMonitorStack.yml new file mode 100644 index 00000000..a433c341 --- /dev/null +++ b/elk_metrics_6x/installMonitorStack.yml @@ -0,0 +1,277 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install MonitorStack + hosts: "nova_compute:utility_all:memcached_all" + become: true + gather_facts: true + vars: + haproxy_ssl: false + monitorstack_distro_packages: + ubuntu: + - gcc + - git + - python-dev + - pkg-config + redhat: + - gcc + - git + - python-devel + suse: + - gcc + - git + - python-devel + - pkg-config + + monitorstack_config_enabled: + - check: kvm + options: '' + condition: >- + {{ + inventory_hostname in (groups['nova_compute'] | default([])) + }} + - check: memcache + options: >- + --host {{ (monitorstack_memcached_access.stdout_lines[0] | default("127.0.0.1:11211")).split(":")[0] }} + --port {{ (monitorstack_memcached_access.stdout_lines[0] | default("127.0.0.1:11211")).split(":")[1] }} + condition: >- + {{ + inventory_hostname in (groups['memcached_all'] | default([])) + }} + - check: os_block_pools_totals + options: '' + condition: >- + {{ + (clouds_config.stat.exists | bool) and + (inventory_hostname in (groups['utility_all'] | default([]))) and + (inventory_hostname == (groups['utility_all'] | default([null]))[0]) + }} + - check: os_block_pools_usage + options: '' + condition: >- + {{ + (clouds_config.stat.exists | bool) and + (inventory_hostname in (groups['utility_all'] | default([]))) and + (inventory_hostname == (groups['utility_all'] | default([null]))[0]) + }} + - check: os_vm_quota_cores + options: '' + condition: >- + {{ + (clouds_config.stat.exists | bool) and + (inventory_hostname in (groups['utility_all'] | default([]))) and + (inventory_hostname == (groups['utility_all'] | default([null]))[0]) + }} + - check: os_vm_quota_instance + options: '' + condition: >- + {{ + (clouds_config.stat.exists | bool) and + (inventory_hostname in (groups['utility_all'] | default([]))) and + (inventory_hostname == (groups['utility_all'] | default([null]))[0]) + }} + - check: os_vm_quota_ram + options: '' + condition: >- + {{ + (clouds_config.stat.exists | bool) and + (inventory_hostname in (groups['utility_all'] | default([]))) and + (inventory_hostname == (groups['utility_all'] | default([null]))[0]) + }} + - check: os_vm_used_cores + options: '' + condition: >- + {{ + (clouds_config.stat.exists | bool) and + (inventory_hostname in (groups['utility_all'] | default([]))) and + (inventory_hostname == (groups['utility_all'] | default([null]))[0]) + }} + - check: os_vm_used_disk + options: '' + condition: >- + {{ + (clouds_config.stat.exists | bool) and + (inventory_hostname in (groups['utility_all'] | default([]))) and + (inventory_hostname == (groups['utility_all'] | default([null]))[0]) + }} + - check: os_vm_used_instance + options: '' + condition: >- + {{ + (clouds_config.stat.exists | bool) and + (inventory_hostname in (groups['utility_all'] | default([]))) and + (inventory_hostname == (groups['utility_all'] | default([null]))[0]) + }} + - check: os_vm_used_ram + options: '' + condition: >- + {{ + (clouds_config.stat.exists | bool) and + (inventory_hostname in (groups['utility_all'] | default([]))) and + (inventory_hostname == (groups['utility_all'] | default([null]))[0]) + }} + - check: uptime + options: '' + condition: true + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + roles: + - role: elastic_data_hosts + + post_tasks: + - name: Find clouds config + stat: + path: "{{ ansible_env.HOME }}/.config/openstack/clouds.yaml" + register: clouds_config + + - name: Find openstack release + stat: + path: "/etc/openstack-release" + register: openstack_release + + - name: Find osp release + stat: + path: "/etc/rhosp-release" + register: rhosp_release + + - name: MonitorStack block + when: + - (openstack_release.stat.exists | bool) or + (rhosp_release.stat.exists | bool) + block: + - name: Ensure disto packages are installed + package: + name: "{{ monitorstack_distro_packages[(ansible_distribution | lower)] }}" + state: "{{ monitorstack_package_state | default('present') }}" + update_cache: "{{ (ansible_pkg_mgr == 'apt') | ternary('yes', omit) }}" + + - name: Refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + tags: + - always + + - name: create the system group + group: + name: "monitorstack" + state: "present" + system: "yes" + + - name: Create the monitorstack system user + user: + name: "monitorstack" + group: "monitorstack" + comment: "monitorstack user" + shell: "/bin/false" + createhome: "yes" + home: "/var/lib/monitorstack" + + - name: Create monitorstack data path + file: + path: "{{ item }}" + state: directory + owner: "monitorstack" + group: "monitorstack" + mode: "0750" + recurse: true + with_items: + - "/var/lib/monitorstack" + - "/var/lib/monitorstack/.config" + - "/var/lib/monitorstack/.config/openstack" + - "/var/lib/monitorstack/venv" + - "/var/log/monitorstack" + - "/etc/monitorstack" + + - name: Copy the clouds config into monitorstack + copy: + src: "{{ ansible_env.HOME }}/.config/openstack/clouds.yaml" + dest: "/var/lib/monitorstack/.config/openstack/clouds.yaml" + remote_src: yes + when: + - clouds_config.stat.exists | bool + + - name: Create the virtualenv (if it does not exist) + command: "virtualenv --no-setuptools --system-site-packages /var/lib/monitorstack/venv" + args: + creates: "/var/lib/monitorstack/venv/bin/activate" + + - name: Setup venv + pip: + name: + - pip + - setuptools + virtualenv_site_packages: yes + extra_args: "-U" + virtualenv: "/var/lib/monitorstack/venv" + + - name: Ensure monitorstack is installed + pip: + name: "git+https://github.com/openstack/monitorstack@{{ monitorstack_release | default('master') }}" + state: "{{ monitorstack_package_state | default('present') }}" + extra_args: --isolated + virtualenv: /var/lib/monitorstack/venv + register: _pip_task + until: _pip_task is success + retries: 3 + delay: 2 + tags: + - package_install + + - name: Create montiorstack config + copy: + dest: "/etc/monitorstack/monitorstack.ini" + content: | + [elasticsearch] + hosts = {{ elasticsearch_data_hosts | join(',') }} + port = {{ elastic_port }} + + - name: Run memcached port scan + shell: "ss -ntlp | awk '/11211/ {print $4}'" + register: monitorstack_memcached_access + changed_when: false + + - name: Run the systemd service role + include_role: + name: systemd_service + vars: + systemd_user_name: monitorstack + systemd_group_name: monitorstack + systemd_services: |- + {% set services = [] %} + {% for item in monitorstack_config_enabled %} + {% if item.condition | bool %} + {% + set check = { + "service_name": ("monitorstack-" ~ item.check), + "execstarts": ("/var/lib/monitorstack/venv/bin/monitorstack --format elasticsearch --config-file /etc/monitorstack/monitorstack.ini " ~ item.check ~ ' ' ~ item.options), + "timer": { + "state": "started", + "options": { + "OnBootSec": "5min", + "OnUnitActiveSec": "10m", + "Persistent": true + } + } + } + %} + {% set _ = services.append(check) %} + {% endif %} + {% endfor %} + {{ services }} + + tags: + - beat-install diff --git a/elk_metrics_6x/roles/elastic_retention/defaults/main.yml b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml index 00a789d9..8b8e1da0 100644 --- a/elk_metrics_6x/roles/elastic_retention/defaults/main.yml +++ b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml @@ -106,6 +106,10 @@ elastic_beat_retention_policy_hosts: timeFieldName: '@timestamp' weight: 1 hosts: "{{ groups['hosts'] | default([]) }}" + monitorstack: + timeFieldName: '@timestamp' + weight: 1 + hosts: "{{ (groups['nova_compute'] | default([])) | union((groups['utility_all'] | default([]))) | union((groups['memcached_all'] | default([]))) }}" skydive: weight: 1 hosts: "{{ (((groups['skydive_analyzers'] | default([])) | length) > 0) | ternary((groups['hosts'] | default([])), []) }}" diff --git a/elk_metrics_6x/site.yml b/elk_metrics_6x/site.yml index 92fbe28b..5b43f570 100644 --- a/elk_metrics_6x/site.yml +++ b/elk_metrics_6x/site.yml @@ -13,4 +13,5 @@ - import_playbook: site-elka.yml - import_playbook: site-beats-core.yml +- import_playbook: installMonitorStack.yml - import_playbook: createElasticIndexes.yml diff --git a/elk_metrics_6x/vars/variables.yml b/elk_metrics_6x/vars/variables.yml index 9f6fc36d..81fc35fa 100644 --- a/elk_metrics_6x/vars/variables.yml +++ b/elk_metrics_6x/vars/variables.yml @@ -366,3 +366,18 @@ grafana_datasources: maxConcurrentShardRequests: 256 timeField: "@timestamp" timeInterval: ">10s" + - name: "monitorstack-Elasticsearch" + type: "elasticsearch" + access: "proxy" + url: "{{ elastic_vip_url }}" + basicAuth: false + basicAuthUser: "" + basicAuthPassword: "" + isDefault: false + database: "monitorstack-*" + jsonData: + esVersion: 56 + keepCookies: [] + maxConcurrentShardRequests: 256 + timeField: "@timestamp" + timeInterval: ">60s"