From 13d5a967591216f64679bcb614e03fd9de0c82fe Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Wed, 4 Jun 2014 22:08:02 +0200 Subject: [PATCH] Implement logging support * Add 3 new puppet modules in dependencies * Configure Logging server: * Configure elasticsearch * Configure Kibana3 * Configure Logging agent: * Configure fluentd * Unit tests Signed-off-by: Emilien Macchi --- .fixtures.yml | 9 ++++ Puppetfile | 9 ++++ manifests/logging.pp | 26 ++++++++++++ manifests/logging/agent.pp | 63 ++++++++++++++++++++++++++++ manifests/logging/server.pp | 50 ++++++++++++++++++++++ spec/classes/cloud_logging_agent.rb | 55 ++++++++++++++++++++++++ spec/classes/cloud_logging_server.rb | 55 ++++++++++++++++++++++++ 7 files changed, 267 insertions(+) create mode 100644 manifests/logging.pp create mode 100644 manifests/logging/agent.pp create mode 100644 manifests/logging/server.pp create mode 100644 spec/classes/cloud_logging_agent.rb create mode 100644 spec/classes/cloud_logging_server.rb diff --git a/.fixtures.yml b/.fixtures.yml index 092824db..5501ea84 100644 --- a/.fixtures.yml +++ b/.fixtures.yml @@ -39,12 +39,21 @@ fixtures: 'concat': repo: 'git://github.com/enovance/puppet-concat.git' ref: '04356974f72b90a1d0f57346a00e95a717924e43' + 'elasticsearch': + repo: 'git://github.com/enovance/puppet-elasticsearch.git' + ref: '042720d52a3879786c2f9d9f67961c4f42004fae' + 'fluentd': + repo: 'git://github.com/enovance/puppet-fluentd.git' + ref: '474ad6d2b84f3d1e03000c2fc340ec1e3408488d' 'haproxy': repo: 'git://github.com/enovance/puppetlabs-haproxy.git' ref: 'ff713f85d8fac7ade808f3d65d949a1a06b8ea88' 'keepalived': repo: 'git://github.com/enovance/puppet-module-keepalived.git' ref: 'eb345b6d3b25106cbe166028f2b8dd9974a10230' + 'kibana3': + repo: 'git://github.com/enovance/kibana3.git' + ref: '977699fa283ebbd593ba0b1ba99791e928681e29' 'mongodb': repo: 'git://github.com/enovance/puppetlabs-mongodb.git' ref: 'e3cc9a0ea895f6e01801e4620799c66c2337b7ed' diff --git a/Puppetfile b/Puppetfile index 265a5819..dbd563d3 100644 --- a/Puppetfile +++ b/Puppetfile @@ -71,6 +71,12 @@ mod 'corosync', mod 'dnsclient', :git => 'git://github.com/enovance/puppet-module-dnsclient.git', :ref => '4158b30f4660623f98dcdbd1ce9b482556180b57' +mod 'elasticsearch', + :git => 'git://github.com/enovance/puppet-elasticsearch.git', + :ref => '042720d52a3879786c2f9d9f67961c4f42004fae' +mod 'fluentd', + :git => 'git://github.com/enovance/puppet-fluentd.git', + :ref => '474ad6d2b84f3d1e03000c2fc340ec1e3408488d' mod 'haproxy', :git => 'git://github.com/enovance/puppetlabs-haproxy.git', :ref => 'ff713f85d8fac7ade808f3d65d949a1a06b8ea88' @@ -80,6 +86,9 @@ mod 'inifile', mod 'keepalived', :git => 'git://github.com/enovance/puppet-module-keepalived.git', :ref => 'eb345b6d3b25106cbe166028f2b8dd9974a10230' +mod 'kibana3', + :git => 'git://github.com/enovance/kibana3.git', + :ref => '977699fa283ebbd593ba0b1ba99791e928681e29' mod 'kmod', :git => 'git://github.com/enovance/puppet-kmod.git', :ref => 'accc40093e6f8ee9cc472e9eb6ba3bab4bad3a1f' diff --git a/manifests/logging.pp b/manifests/logging.pp new file mode 100644 index 00000000..08a73ab3 --- /dev/null +++ b/manifests/logging.pp @@ -0,0 +1,26 @@ +# +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# == Class: cloud::logging +# +# Configure common logging +# + +class cloud::logging{ + + include ::fluentd + fluentd::configfile { 'forward': } + +} diff --git a/manifests/logging/agent.pp b/manifests/logging/agent.pp new file mode 100644 index 00000000..fbf4ae56 --- /dev/null +++ b/manifests/logging/agent.pp @@ -0,0 +1,63 @@ +# +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# == Class: cloud::logging::agent +# +# Configure logging agent +# +# === Parameters: +# +# [*server*] +# (optional) IP address or hostname of the logging server +# Defaults to '127.0.0.1' +# + +class cloud::logging::agent( + $server = '127.0.0.1' +){ + + include cloud::logging + + fluentd::configfile { 'syslog': } + + file { '/var/db': + ensure => directory, + } -> + file { '/var/db/td-agent': + ensure => 'directory', + owner => 'td-agent', + group => 'td-agent', + require => Class['fluentd'], + } + + fluentd::source { 'syslog_main': + configfile => 'syslog', + type => 'tail', + format => 'syslog', + tag => 'system.syslog', + config => { + path => '/var/log/syslog', + pos_file => '/var/db/td-agent/td-agent.syslog.pos' + } + } + + fluentd::match { 'forward_main': + configfile => 'forward', + pattern => '**', + type => 'forward', + servers => [ { 'host' => $server } ] + } + +} diff --git a/manifests/logging/server.pp b/manifests/logging/server.pp new file mode 100644 index 00000000..cc2e8b42 --- /dev/null +++ b/manifests/logging/server.pp @@ -0,0 +1,50 @@ +# +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# == Class: cloud::logging::server +# + +class cloud::logging::server{ + + include cloud::logging + + class { 'elasticsearch': + config => {} + } + + # kibana3 requires a separate vhost or a different port + class { 'kibana3': + ws_port => 8001, + } + + fluentd::install_plugin { 'elasticsearch-plugin': + ensure => present, + plugin_type => 'gem', + plugin_name => 'fluent-plugin-elasticsearch', + } + + fluentd::source { 'forward_collector': + configfile => 'forward', + type => 'forward', + } + + fluentd::match { 'forward_logs': + configfile => 'forward', + pattern => '**', + type => 'elasticsearch', + config => { logstash_format => true } + } + +} diff --git a/spec/classes/cloud_logging_agent.rb b/spec/classes/cloud_logging_agent.rb new file mode 100644 index 00000000..a8c94ef9 --- /dev/null +++ b/spec/classes/cloud_logging_agent.rb @@ -0,0 +1,55 @@ +# +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Unit tests for cloud::logging::agent class +# + +require 'spec_helper' + +describe 'cloud::logging::agent' do + + shared_examples_for 'openstack logging server' do + + let :pre_condition do + "class { 'cloud::logging': } + include ::fluentd" + end + + it 'configure logging common' do + it should contain_concat("/etc/td-agent/config.d/forward.conf") + end + + it 'configure elasticsearch' do + should contain_class('elasticsearch') + end + end + + context 'on Debian platforms' do + let :facts do + { :osfamily => 'Debian' } + end + + it_configures 'openstack logging server' + end + + context 'on RedHat platforms' do + let :facts do + { :osfamily => 'RedHat' } + end + + it_configures 'openstack logging server' + end + +end diff --git a/spec/classes/cloud_logging_server.rb b/spec/classes/cloud_logging_server.rb new file mode 100644 index 00000000..17bec47a --- /dev/null +++ b/spec/classes/cloud_logging_server.rb @@ -0,0 +1,55 @@ +# +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Unit tests for cloud::logging::server class +# + +require 'spec_helper' + +describe 'cloud::logging::server' do + + shared_examples_for 'openstack logging server' do + + let :pre_condition do + "class { 'cloud::logging': } + include ::fluentd" + end + + it 'configure logging common' do + it should contain_concat("/etc/td-agent/config.d/forward.conf") + end + + it 'configure kibana' do + should contain_class('kibana3').with(:ws_port => '8001') + end + end + + context 'on Debian platforms' do + let :facts do + { :osfamily => 'Debian' } + end + + it_configures 'openstack logging server' + end + + context 'on RedHat platforms' do + let :facts do + { :osfamily => 'RedHat' } + end + + it_configures 'openstack logging server' + end + +end