Implement logging support

* Add 3 new puppet modules in dependencies
* Configure Logging server:
  * Configure elasticsearch
  * Configure Kibana3
* Configure Logging agent:
  * Configure fluentd
* Unit tests

Signed-off-by: Emilien Macchi <emilien.macchi@enovance.com>

.fixtures.yml

@@ -39,12 +39,21 @@ fixtures:
     'concat':
       repo: 'git://github.com/enovance/puppet-concat.git'
       ref: '04356974f72b90a1d0f57346a00e95a717924e43'
+    'elasticsearch':
+      repo: 'git://github.com/enovance/puppet-elasticsearch.git'
+      ref: '042720d52a3879786c2f9d9f67961c4f42004fae'
+    'fluentd':
+      repo: 'git://github.com/enovance/puppet-fluentd.git'
+      ref: '474ad6d2b84f3d1e03000c2fc340ec1e3408488d'
     'haproxy':
       repo: 'git://github.com/enovance/puppetlabs-haproxy.git'
       ref: 'ff713f85d8fac7ade808f3d65d949a1a06b8ea88'
     'keepalived':
       repo: 'git://github.com/enovance/puppet-module-keepalived.git'
       ref: 'eb345b6d3b25106cbe166028f2b8dd9974a10230'
+    'kibana3':
+      repo: 'git://github.com/enovance/kibana3.git'
+      ref: '977699fa283ebbd593ba0b1ba99791e928681e29'
     'mongodb':
       repo: 'git://github.com/enovance/puppetlabs-mongodb.git'
       ref: 'e3cc9a0ea895f6e01801e4620799c66c2337b7ed'

Puppetfile

@@ -71,6 +71,12 @@ mod 'corosync',
 mod 'dnsclient',
   :git => 'git://github.com/enovance/puppet-module-dnsclient.git',
   :ref => '4158b30f4660623f98dcdbd1ce9b482556180b57'
+mod 'elasticsearch',
+  :git => 'git://github.com/enovance/puppet-elasticsearch.git',
+  :ref => '042720d52a3879786c2f9d9f67961c4f42004fae'
+mod 'fluentd',
+  :git => 'git://github.com/enovance/puppet-fluentd.git',
+  :ref => '474ad6d2b84f3d1e03000c2fc340ec1e3408488d'
 mod 'haproxy',
   :git => 'git://github.com/enovance/puppetlabs-haproxy.git',
   :ref => 'ff713f85d8fac7ade808f3d65d949a1a06b8ea88'
@@ -80,6 +86,9 @@ mod 'inifile',
 mod 'keepalived',
   :git => 'git://github.com/enovance/puppet-module-keepalived.git',
   :ref => 'eb345b6d3b25106cbe166028f2b8dd9974a10230'
+mod 'kibana3',
+  :git => 'git://github.com/enovance/kibana3.git',
+  :ref => '977699fa283ebbd593ba0b1ba99791e928681e29'
 mod 'kmod',
   :git => 'git://github.com/enovance/puppet-kmod.git',
   :ref => 'accc40093e6f8ee9cc472e9eb6ba3bab4bad3a1f'

manifests/logging.pp

@@ -0,0 +1,26 @@
+#
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Class: cloud::logging
+#
+# Configure common logging
+#
+
+class cloud::logging{
+
+  include ::fluentd
+  fluentd::configfile { 'forward': }
+
+}

manifests/logging/agent.pp

@@ -0,0 +1,63 @@
+#
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Class: cloud::logging::agent
+#
+# Configure logging agent
+#
+# === Parameters:
+#
+# [*server*]
+#   (optional) IP address or hostname of the logging server
+#   Defaults to '127.0.0.1'
+#
+
+class cloud::logging::agent(
+  $server = '127.0.0.1'
+){
+
+  include cloud::logging
+
+  fluentd::configfile { 'syslog': }
+
+  file { '/var/db':
+    ensure => directory,
+  } ->
+  file { '/var/db/td-agent':
+    ensure  => 'directory',
+    owner   => 'td-agent',
+    group   => 'td-agent',
+    require => Class['fluentd'],
+  }
+
+  fluentd::source { 'syslog_main':
+    configfile => 'syslog',
+    type       => 'tail',
+    format     => 'syslog',
+    tag        => 'system.syslog',
+    config     => {
+      path     => '/var/log/syslog',
+      pos_file => '/var/db/td-agent/td-agent.syslog.pos'
+    }
+  }
+
+  fluentd::match { 'forward_main':
+    configfile => 'forward',
+    pattern    => '**',
+    type       => 'forward',
+    servers    => [ { 'host' => $server } ]
+  }
+
+}

manifests/logging/server.pp

@@ -0,0 +1,50 @@
+#
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Class: cloud::logging::server
+#
+
+class cloud::logging::server{
+
+  include cloud::logging
+
+  class { 'elasticsearch':
+    config => {}
+  }
+
+  # kibana3 requires a separate vhost or a different port
+  class { 'kibana3':
+    ws_port => 8001,
+  }
+
+  fluentd::install_plugin { 'elasticsearch-plugin':
+    ensure      => present,
+    plugin_type => 'gem',
+    plugin_name => 'fluent-plugin-elasticsearch',
+  }
+
+  fluentd::source { 'forward_collector':
+    configfile => 'forward',
+    type       => 'forward',
+  }
+
+  fluentd::match { 'forward_logs':
+    configfile => 'forward',
+    pattern    => '**',
+    type       => 'elasticsearch',
+    config     => { logstash_format => true }
+  }
+
+}

spec/classes/cloud_logging_agent.rb

@@ -0,0 +1,55 @@
+#
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Unit tests for cloud::logging::agent class
+#
+
+require 'spec_helper'
+
+describe 'cloud::logging::agent' do
+
+  shared_examples_for 'openstack logging server' do
+
+    let :pre_condition do
+      "class { 'cloud::logging': }
+      include ::fluentd"
+    end
+
+    it 'configure logging common' do
+      it should contain_concat("/etc/td-agent/config.d/forward.conf")
+    end
+
+    it 'configure elasticsearch' do
+      should contain_class('elasticsearch')
+    end
+  end
+
+  context 'on Debian platforms' do
+    let :facts do
+      { :osfamily => 'Debian' }
+    end
+
+    it_configures 'openstack logging server'
+  end
+
+  context 'on RedHat platforms' do
+    let :facts do
+      { :osfamily => 'RedHat' }
+    end
+
+    it_configures 'openstack logging server'
+  end
+
+end

spec/classes/cloud_logging_server.rb

@@ -0,0 +1,55 @@
+#
+# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Unit tests for cloud::logging::server class
+#
+
+require 'spec_helper'
+
+describe 'cloud::logging::server' do
+
+  shared_examples_for 'openstack logging server' do
+
+    let :pre_condition do
+      "class { 'cloud::logging': }
+      include ::fluentd"
+    end
+
+    it 'configure logging common' do
+      it should contain_concat("/etc/td-agent/config.d/forward.conf")
+    end
+
+    it 'configure kibana' do
+      should contain_class('kibana3').with(:ws_port => '8001')
+    end
+  end
+
+  context 'on Debian platforms' do
+    let :facts do
+      { :osfamily => 'Debian' }
+    end
+
+    it_configures 'openstack logging server'
+  end
+
+  context 'on RedHat platforms' do
+    let :facts do
+      { :osfamily => 'RedHat' }
+    end
+
+    it_configures 'openstack logging server'
+  end
+
+end