logging: Make the logging infrastructure more flexible

Since the logcollector is an agent itself, it inherits from the agent role.
Elasticsearch and Kibana are configurable via the hiera file.

.fixtures.yml

@@ -99,6 +99,9 @@ fixtures:
     'ssh':
       repo: 'git://github.com/enovance/puppet-ssh.git'
       ref: '10675c0d80511a8cdd514af67b695887fa97ec40'
+    'rsyslog':
+      repo: 'git://github.com/enovance/puppet-rsyslog.git'
+      ref: '3072553c9543b5c7769c54ed251bdfcce2967ce9'
     'rsync':
       repo: 'git://github.com/enovance/puppetlabs-rsync.git'
       ref: '7122983d89bf68bc4170415cc03212f6a8a4636e'

Puppetfile

@@ -122,6 +122,9 @@ mod 'rsync',
 mod 'ssh',
   :git => 'git://github.com/enovance/puppet-ssh.git',
   :ref => '10675c0d80511a8cdd514af67b695887fa97ec40'
+mod 'rsyslog',
+  :git => 'git://github.com/enovance/puppet-rsyslog.git',
+  :ref => '3072553c9543b5c7769c54ed251bdfcce2967ce9'
 mod 'stdlib',
   :git => 'git://github.com/enovance/puppetlabs-stdlib.git',
   :ref => '224b8f9a191f635b03ee900a9bf87bfdb0f1a6ed'

manifests/logging/agent.pp

@@ -19,25 +19,36 @@
 #
 # === Parameters:
 #
-# [*server*]
+# [*syslog_enable*]
-#   (optional) IP address or hostname of the logging server
+#   (optional) Enable the configuration of rsyslog
-#   Defaults to '127.0.0.1'
+#   Defaults to false
 #
 # [*sources*]
 #   (optional) Fluentd sources
 #   Defaults to empty hash
 #
+# [*matches*]
+#   (optional) Fluentd matches
+#   Defaults to empty hash
+#
+# [*plugins*]
+#   (optional) Fluentd plugins to install
+#   Defaults to empty hash
+#
 class cloud::logging::agent(
-  $server = '127.0.0.1',
+  $syslog_enable = false,
-  $sources = {},
+  $sources       = {},
+  $matches       = {},
+  $plugins       = {},
 ){
 
   include cloud::logging
 
-  resources {'fluentd::configfile':
+  if $syslog_enable {
-    purge => true,
+    include rsyslog::client
   }
-  resources {'fluentd::source' :
+
+  resources {['fluentd::configfile', 'fluentd::source', 'fluentd::match']:
     purge => true,
   }
 
@@ -51,14 +62,10 @@ class cloud::logging::agent(
     require => Class['fluentd'],
   }
 
-  create_resources('fluentd::configfile', keys($sources))
+  ensure_resource('fluentd::configfile', keys($sources))
-  create_resources('fluentd::source', $sources)
+  ensure_resource('fluentd::configfile', keys($matches))
-
+  create_resources('fluentd::source', $sources, {'require' => 'File[\'/var/db/td-agent\']'})
-  fluentd::match { 'forward_main':
+  create_resources('fluentd::match', $matches)
-    configfile => 'forward',
+  create_resources('fluentd::install_plugin', $plugins)
-    pattern    => '**',
-    type       => 'forward',
-    servers    => [ { 'host' => $server } ]
-  }
 
 }

manifests/logging/server.pp

@@ -18,33 +18,8 @@
 
 class cloud::logging::server{
 
-  include cloud::logging
+  include ::elasticsearch
-
+  include ::kibana3
-  class { 'elasticsearch':
+  include cloud::logging::agent
-    config => {}
-  }
-
-  # kibana3 requires a separate vhost or a different port
-  class { 'kibana3':
-    ws_port => 8001,
-  }
-
-  fluentd::install_plugin { 'elasticsearch-plugin':
-    ensure      => present,
-    plugin_type => 'gem',
-    plugin_name => 'fluent-plugin-elasticsearch',
-  }
-
-  fluentd::source { 'forward_collector':
-    configfile => 'forward',
-    type       => 'forward',
-  }
-
-  fluentd::match { 'forward_logs':
-    configfile => 'forward',
-    pattern    => '**',
-    type       => 'elasticsearch',
-    config     => { logstash_format => true }
-  }
 
 }

spec/classes/cloud_logging_agent.rb

@@ -20,14 +20,14 @@ require 'spec_helper'
 
 describe 'cloud::logging::agent' do
 
-  shared_examples_for 'openstack logging server' do
+  shared_examples_for 'openstack logging agent' do
 
     let :pre_condition do
       "class { 'cloud::logging': }
       include ::fluentd"
     end
 
-    let :params do {
+    let :common_params do {
       :server => '127.0.0.1',
       :sources => {
         'apache' => {'type' => 'tail', 'configfile' => 'apache'},
@@ -36,16 +36,50 @@ describe 'cloud::logging::agent' do
     }
     end
 
-    it 'configure logging common' do
+
-      it should contain_concat("/etc/td-agent/config.d/forward.conf")
+    context 'rsyslog is enabled' do
+      let :params do 
+        common_params.merge( {:syslog_enable => 'true' } )
+      end
+
+      it 'include cloud::loging' do
+        it should contain_class('cloud::logging')
+      end
+
+      it 'include rsyslog::client' do
+        it should contain_class('rsyglog::client')
+      end
+
+      it 'create /var/db/td-agent' do
+        it should contain_file('/var/db/td-agent').with({
+          :ensure => 'directory',
+          :owner  => 'td-agent',
+          :group  => 'td-agent',
+        })
+      end
+
     end
 
-    it 'config apache logging source' do
+    context 'rsyslog is disabled' do
-      it should contain_fluentd__configfile('apache')
+      let :params do 
-      it should contain_fluentd__source('apache').with({
+        common_params.merge( {:syslog_enable => 'false' } )
-        :type       => 'tail',
+      end
-        :configfile => 'apache',
+
-      })
+      it 'include cloud::loging' do
+        it should contain_class('cloud::logging')
+      end
+
+      it 'include rsyslog::client' do
+        it should_not contain_class('rsyglog::client')
+      end
+
+      it 'create /var/db/td-agent' do
+        it should contain_file('/var/db/td-agent').with({
+          :ensure => 'directory',
+          :owner  => 'td-agent',
+          :group  => 'td-agent',
+        })
+      end
     end
 
   end
@@ -55,7 +89,7 @@ describe 'cloud::logging::agent' do
       { :osfamily => 'Debian' }
     end
 
-    it_configures 'openstack logging server'
+    it_configures 'openstack logging agent'
   end
 
   context 'on RedHat platforms' do
@@ -63,7 +97,7 @@ describe 'cloud::logging::agent' do
       { :osfamily => 'RedHat' }
     end
 
-    it_configures 'openstack logging server'
+    it_configures 'openstack logging agent'
   end
 
 end