diff --git a/elk_metrics_6x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 b/elk_metrics_6x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 index 0ca51c7a..c00ac6fc 100644 --- a/elk_metrics_6x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 +++ b/elk_metrics_6x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 @@ -1,78 +1,62 @@ {% import 'templates/_macros.j2' as elk_macros %} -#======================== Journalbeat Configuration ============================ +###################### Journalbeat Configuration Example ######################### +# This file is an example configuration file highlighting only the most common +# options. The journalbeat.reference.yml file from the same directory contains all the +# supported options with more comments. You can use it as a reference. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/journalbeat/index.html + +# For more available modules and options, please see the journalbeat.reference.yml sample +# configuration file. + +#=========================== Journalbeat inputs ============================= + +journalbeat.inputs: + # Paths that should be crawled and fetched. Possible values files and directories. + # When setting a directory, all journals under it are merged. + # When empty starts to read from local journal. +- paths: ["/var/log/journal"] + + # The number of seconds to wait before trying to read again from journals. + #backoff: 1s + # The maximum number of seconds to wait before attempting to read again from journals. + #max_backoff: 60s + + # Position to start reading from journal. Valid values: head, tail, cursor + seek: cursor + # Fallback position if no cursor data is available. + #cursor_seek_fallback: head + + # Exact matching for field values of events. + # Matching for nginx entries: "systemd.unit=nginx" + #include_matches: [] + + # Optional fields that you can specify to add additional information to the + # output. Fields can be scalar values, arrays, dictionaries, or any nested + # combination of these. + #fields: + # env: staging + + +#========================= Journalbeat global options ============================ journalbeat: - # What position in journald to seek to at start up - # options: cursor, tail, head (defaults to tail) - #seek_position: tail + # Name of the registry file. If a relative path is used, it is considered relative to the + # data path. + registry_file: registry - # If seek_position is set to cursor and seeking to cursor fails - # fall back to this method. If set to none will it will exit - # options: tail, head, none (defaults to tail) - #cursor_seek_fallback: tail + # The number of seconds to wait before trying to read again from journals. + backoff: 10s + # The maximum number of seconds to wait before attempting to read again from journals. + max_backoff: 60s - # Store the cursor of the successfully published events - #write_cursor_state: true + # Position to start reading from all journal. Possible values: head, tail, cursor + seek: head - # Path to the file to store the cursor (defaults to ".journalbeat-cursor-state") - #cursor_state_file: .journalbeat-cursor-state - - # How frequently should we save the cursor to disk (defaults to 5s) - cursor_flush_period: 10s - - # Path to the file to store the queue of events pending (defaults to ".journalbeat-pending-queue") - #pending_queue.file: .journalbeat-pending-queue - - # How frequently should we save the queue to disk (defaults to 1s). - # Pending queue represents the WAL of events queued to be published - # or being published and waiting for acknowledgement. In case of a - # regular restart of journalbeat all the events not yet acknowledged - # will be flushed to disk during the shutdown. - # In case of disaster most probably journalbeat won't get a chance to shutdown - # itself gracefully and this flush period option will serve you as a - # backup creation frequency option. - pending_queue.flush_period: 5s - - # Size of the buffered queue for the published and acknowledged messages - #pending_queue.completed_queue_size: 8192 - - # Lowercase and remove leading underscores, e.g. "_MESSAGE" -> "message" - # (defaults to false) - clean_field_names: true - - # All journal entries are strings by default. You can try to convert them to numbers. - # (defaults to false) - #convert_to_numbers: false - - # Store all the fields of the Systemd Journal entry under this field - # Can be almost any string suitable to be a field name of an ElasticSearch document. - # Dots can be used to create nested fields. - # Two exceptions: - # - no repeated dots; - # - no trailing dots, e.g. "journal..field_name." will fail - # (defaults to "" hence stores on the upper level of the event) - #move_metadata_to_field: "" - - # Specific units to monitor. - #units: ["httpd.service"] - - # gather kernel logs when units are provided - kernel: true - - # Custom Journal patterns to match on other than UNIT - #match_patterns: ["FIELD=value"] - - # Specificies syslog identifiers to monitor. - #identifiers: ["docker"] - - # Specify Journal paths to open. You can pass an array of paths to Systemd Journal paths. - # If you want to open Journal from directory just pass an array consisting of one element - # representing the path. See: https://www.freedesktop.org/software/systemd/man/sd_journal_open.html - # By default this setting is empty thus journalbeat will attempt to find all journal files automatically -{% if journal_dir.stat.exists and journal_dir.stat.isdir %} - journal_paths: ["/var/log/journal"] -{% endif %} - #default_type: journal + # Exact matching for field values of events. + # Matching for nginx entries: "systemd.unit=nginx" + #matches: [] #================================ General ====================================== @@ -98,12 +82,86 @@ tags: # sub-dictionary. Default is false. #fields_under_root: false -# Internal queue size for single events in processing pipeline -#queue_size: 1000 +# Internal queue configuration for buffering events to be published. +#queue: + # Queue type by name (default 'mem') + # The memory queue will present all available events (up to the outputs + # bulk_max_size) to the output, the moment the output is ready to server + # another batch of events. + #mem: + # Max number of events the queue can buffer. + #events: 4096 -# The internal queue size for bulk events in the processing pipeline. -# Do not modify this value. -#bulk_queue_size: 0 + # Hints the minimum number of events stored in the queue, + # before providing a batch of events to the outputs. + # The default value is set to 2048. + # A value of 0 ensures events are immediately available + # to be sent to the outputs. + #flush.min_events: 2048 + + # Maximum duration after which events are available to the outputs, + # if the number of events stored in the queue is < min_flush_events. + #flush.timeout: 1s + + # The spool queue will store events in a local spool file, before + # forwarding the events to the outputs. + # + # Beta: spooling to disk is currently a beta feature. Use with care. + # + # The spool file is a circular buffer, which blocks once the file/buffer is full. + # Events are put into a write buffer and flushed once the write buffer + # is full or the flush_timeout is triggered. + # Once ACKed by the output, events are removed immediately from the queue, + # making space for new events to be persisted. + #spool: + # The file namespace configures the file path and the file creation settings. + # Once the file exists, the `size`, `page_size` and `prealloc` settings + # will have no more effect. + #file: + # Location of spool file. The default value is ${path.data}/spool.dat. + #path: "${path.data}/spool.dat" + + # Configure file permissions if file is created. The default value is 0600. + #permissions: 0600 + + # File size hint. The spool blocks, once this limit is reached. The default value is 100 MiB. + #size: 100MiB + + # The files page size. A file is split into multiple pages of the same size. The default value is 4KiB. + #page_size: 4KiB + + # If prealloc is set, the required space for the file is reserved using + # truncate. The default value is true. + #prealloc: true + + # Spool writer settings + # Events are serialized into a write buffer. The write buffer is flushed if: + # - The buffer limit has been reached. + # - The configured limit of buffered events is reached. + # - The flush timeout is triggered. + #write: + # Sets the write buffer size. + #buffer_size: 1MiB + + # Maximum duration after which events are flushed, if the write buffer + # is not full yet. The default value is 1s. + #flush.timeout: 1s + + # Number of maximum buffered events. The write buffer is flushed once the + # limit is reached. + #flush.events: 16384 + + # Configure the on-disk event encoding. The encoding can be changed + # between restarts. + # Valid encodings are: json, ubjson, and cbor. + #codec: cbor + #read: + # Reader flush timeout, waiting for more events to become available, so + # to fill a complete batch, as required by the outputs. + # If flush_timeout is 0, all available events are forwarded to the + # outputs immediately. + # The default value is 0s. + #flush.timeout: 0s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. @@ -118,8 +176,8 @@ tags: # # event -> filter1 -> event1 -> filter2 ->event2 ... # -# The supported processors are drop_fields, drop_event, include_fields, and -# add_cloud_metadata. +# The supported processors are drop_fields, drop_event, include_fields, +# decode_json_fields, and add_cloud_metadata. # # For example, you can use the following processors to keep the fields that # contain CPU load percentages, but remove the fields that contain CPU ticks @@ -139,12 +197,101 @@ tags: # equals: # http.code: 200 # -# The following example enriches each event with metadata from the cloud -# provider about the host machine. It works on EC2, GCE, and DigitalOcean. +# The following example renames the field a to b: # #processors: -#- add_cloud_metadata: +#- rename: +# fields: +# - from: "a" +# to: "b" # +# The following example tokenizes the string into fields: +# +#processors: +#- dissect: +# tokenizer: "%{key1} - %{key2}" +# field: "message" +# target_prefix: "dissect" +# +# The following example enriches each event with metadata from the cloud +# provider about the host machine. It works on EC2, GCE, DigitalOcean, +# Tencent Cloud, and Alibaba Cloud. +# +#processors: +#- add_cloud_metadata: ~ +# +# The following example enriches each event with the machine's local time zone +# offset from UTC. +# +#processors: +#- add_locale: +# format: offset +# +# The following example enriches each event with docker metadata, it matches +# given fields to an existing container id and adds info from that container: +# +#processors: +#- add_docker_metadata: +# host: "unix:///var/run/docker.sock" +# match_fields: ["system.process.cgroup.id"] +# match_pids: ["process.pid", "process.ppid"] +# match_source: true +# match_source_index: 4 +# match_short_id: false +# cleanup_timeout: 60 +# labels.dedot: false +# # To connect to Docker over TLS you must specify a client and CA certificate. +# #ssl: +# # certificate_authority: "/etc/pki/root/ca.pem" +# # certificate: "/etc/pki/client/cert.pem" +# # key: "/etc/pki/client/cert.key" +# +# The following example enriches each event with docker metadata, it matches +# container id from log path available in `source` field (by default it expects +# it to be /var/lib/docker/containers/*/*.log). +# +#processors: +#- add_docker_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false +# +# The following example enriches each event with process metadata using +# process IDs included in the event. +# +#processors: +#- add_process_metadata: +# match_pids: ["system.process.ppid"] +# target: system.process.parent +# +# The following example decodes fields containing JSON strings +# and replaces the strings with valid JSON objects. +# +#processors: +#- decode_json_fields: +# fields: ["field1", "field2", ...] +# process_array: false +# max_depth: 1 +# target: "" +# overwrite_keys: false +processors: +- add_host_metadata: ~ + +#============================= Elastic Cloud ================================== + +# These settings simplify using journalbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: #================================ Outputs ====================================== @@ -160,12 +307,19 @@ tags: # Scheme and port can be left out and will be set to the default (http and 9200) # In case you specify and additional path, the scheme is required: http://localhost:9200/path # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 + #hosts: ["localhost:9200"] - #hosts: localhost:9200 + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "journalbeat" + #ilm.pattern: "{now/d}-000001" # Set gzip compression level. #compression_level: 0 + # Configure escaping html symbols in strings. + #escape_html: true + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" @@ -179,9 +333,10 @@ tags: # Number of workers per Elasticsearch host. #worker: 1 - # Optional index name. The default is "beatname" plus date - # and generates [beatname-]YYYY.MM.DD keys. - #index: "beatname-%{+yyyy.MM.dd}" + # Optional index name. The default is "journalbeat" plus date + # and generates [journalbeat-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. + #index: "journalbeat-%{[beat.version]}-%{+yyyy.MM.dd}" # Optional ingest node pipeline. By default no pipeline will be used. #pipeline: "" @@ -205,47 +360,21 @@ tags: # The default is 50. #bulk_max_size: 50 - # Configure http request timeout before failing an request to Elasticsearch. + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure http request timeout before failing a request to Elasticsearch. #timeout: 90 - # The number of seconds to wait for new events between two bulk API index requests. - # If `bulk_max_size` is reached before this interval expires, addition bulk index - # requests are made. - #flush_interval: 1s - - # A template is used to set the mapping in Elasticsearch - # By default template loading is enabled and the template is loaded. - # These settings can be adjusted to load your own template or overwrite existing ones. - - # Set to false to disable template loading. - #template.enabled: true - - # Template name. By default the template name is beatname. - #template.name: "beatname" - - # Path to template file - #template.path: "${path.config}/beatname.template.json" - - # Overwrite existing template - #template.overwrite: false - - # If set to true, beatname checks the Elasticsearch version at connect time, and if it - # is 2.x, it loads the file specified by the template.versions.2x.path setting. The - # default is true. - #template.versions.2x.enabled: true - - # Path to the Elasticsearch 2.x version of the template file. - #template.versions.2x.path: "${path.config}/beatname.template-es2x.json" - - # If set to true, beatname checks the Elasticsearch version at connect time, and if it - # is 6.x, it loads the file specified by the template.versions.6x.path setting. The - # default is true. - #template.versions.6x.enabled: true - - # Path to the Elasticsearch 6.x version of the template file. - #template.versions.6x.path: "${path.config}/beatname.template-es6x.json" - - # Use SSL settings for HTTPS. Default is true. + # Use SSL settings for HTTPS. #ssl.enabled: true # Configure SSL verification mode. If `none` is configured, all server hosts @@ -277,6 +406,9 @@ tags: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never #----------------------------- Logstash output --------------------------------- {{ elk_macros.output_logstash(inventory_hostname, logstash_data_hosts, ansible_processor_count, 'journalbeat') }} @@ -316,9 +448,16 @@ tags: #username: '' #password: '' - # Kafka version beatname is assumed to run against. Defaults to the oldest - # supported stable version (currently version 0.8.2.0) - #version: 0.8.2 + # Kafka version journalbeat is assumed to run against. Defaults to the "1.0.0". + #version: '1.0.0' + + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true # Metadata update configuration. Metadata do contain leader information # deciding which broker to use when publishing. @@ -366,6 +505,10 @@ tags: # default is gzip. #compression: gzip + # Set the compression level. Currently only gzip provides a compression level + # between 0 and 9. The default value is chosen by the compression algorithm. + #compression_level: 4 + # The maximum permitted size of JSON-encoded messages. Bigger messages will be # dropped. The default value is 1000000 (bytes). This value should be equal to # or less than the broker's message.max.bytes. @@ -377,9 +520,6 @@ tags: # on error. #required_acks: 1 - # The number of seconds to wait for new events between two producer API calls. - #flush_interval: 1s - # The configurable ClientID used for logging, debugging, and auditing # purposes. The default is "beats". #client_id: beats @@ -416,11 +556,23 @@ tags: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + #------------------------------- Redis output ---------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # The list of Redis servers to connect to. If load balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. @@ -431,8 +583,8 @@ tags: #port: 6379 # The name of the Redis list or channel the events are published to. The - # default is beatname. - #key: beatname + # default is journalbeat. + #key: journalbeat # The password to authenticate with. The default is no authentication. #password: @@ -468,6 +620,17 @@ tags: # until all events are published. The default is 3. #max_retries: 3 + # The number of seconds to wait before trying to reconnect to Redis + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Redis after a network error. The default is 60s. + #backoff.max: 60s + # The maximum number of events to bulk in a single Redis request or pipeline. # The default is 2048. #bulk_max_size: 2048 @@ -513,22 +676,33 @@ tags: # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never #------------------------------- File output ----------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + # Path to the directory where to save the generated files. The option is # mandatory. - #path: "/tmp/beatname" + #path: "/tmp/journalbeat" - # Name of the generated files. The default is `beatname` and it generates - # files: `beatname`, `beatname.1`, `beatname.2`, etc. - #filename: beatname + # Name of the generated files. The default is `journalbeat` and it generates + # files: `journalbeat`, `journalbeat.1`, `journalbeat.2`, etc. + #filename: journalbeat # Maximum size in kilobytes of each file. When this size is reached, and on - # every beatname restart, the files are rotated. The default value is 10240 + # every journalbeat restart, the files are rotated. The default value is 10240 # kB. #rotate_every_kb: 10000 @@ -537,41 +711,53 @@ tags: # default is 7 files. #number_of_files: 7 + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + #----------------------------- Console output --------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true - # Pretty print json event - #pretty: false + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true #================================= Paths ====================================== -# The home path for the beatname installation. This is the default base path +# The home path for the journalbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the # distribution (for example, the sample dashboards). # If not set by a CLI flag or in the configuration file, the default for the # home path is the location of the binary. #path.home: -# The configuration path for the beatname installation. This is the default +# The configuration path for the journalbeat installation. This is the default # base path for configuration files, including the main YAML configuration file # and the Elasticsearch template file. If not set by a CLI flag or in the # configuration file, the default for the configuration path is the home path. #path.config: ${path.home} -# The data path for the beatname installation. This is the default base path -# for all the files in which beatname needs to store its data. If not set by a +# The data path for the journalbeat installation. This is the default base path +# for all the files in which journalbeat needs to store its data. If not set by a # CLI flag or in the configuration file, the default for the data path is a data # subdirectory inside the home path. #path.data: ${path.home}/data -# The logs path for a beatname installation. This is the default location for +# The logs path for a journalbeat installation. This is the default location for # the Beat's log files. If not set by a CLI flag or in the configuration file, # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs +#================================ Keystore ========================================== +# Location of the Keystore containing the keys and their sensitive values. +#keystore.path: "${path.config}/beats.keystore" + #============================== Dashboards ===================================== {{ elk_macros.setup_dashboards('journalbeat') }} @@ -585,3 +771,26 @@ tags: #================================ Logging ====================================== {{ elk_macros.beat_logging('journalbeat') }} + +#============================== Xpack Monitoring =============================== +{{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} + +#================================ HTTP Endpoint ====================================== +# Each beat can expose internal metrics through a HTTP endpoint. For security +# reasons the endpoint is disabled by default. This feature is currently experimental. +# Stats can be access through http://localhost:5066/stats . For pretty JSON output +# append ?pretty to the URL. + +# Defines if the HTTP endpoint is enabled. +#http.enabled: false + +# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. +#http.host: localhost + +# Port on which the HTTP endpoint will bind. Default is 5066. +#http.port: 5066 + +#============================= Process Security ================================ + +# Enable or disable seccomp system call filtering on Linux. Default is enabled. +#seccomp.enabled: true