diff --git a/chef/cookbooks/collectd/attributes/default.rb b/chef/cookbooks/collectd/attributes/default.rb index 16575d1..ef211b5 100644 --- a/chef/cookbooks/collectd/attributes/default.rb +++ b/chef/cookbooks/collectd/attributes/default.rb @@ -41,6 +41,7 @@ default[:collectd][:plugins] = {"cpu"=>{}, "memory"=>"", "match_regex"=>"" } +default[:collectd][:included_plugins] = {"kairosdb"=>{}} default[:collectd][:server][:host] = "10.145.81.250" default[:collectd][:server][:port] = "4242" default[:collectd][:server][:protocol] = "tcp" diff --git a/chef/cookbooks/collectd/files/default/rabbitmq_info.py b/chef/cookbooks/collectd/files/default/rabbitmq_info.py new file mode 100644 index 0000000..fe93e45 --- /dev/null +++ b/chef/cookbooks/collectd/files/default/rabbitmq_info.py @@ -0,0 +1,150 @@ +# Name: rabbitmq-collectd-plugin - rabbitmq_info.py +# Author: https://github.com/phrawzty/rabbitmq-collectd-plugin/commits/master +# Description: This plugin uses Collectd's Python plugin to obtain RabbitMQ metrics. +# +# Copyright 2012 Daniel Maher +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## copied from https://github.com/phrawzty/rabbitmq-collectd-plugin + +import collectd +import subprocess +import re + + +NAME = 'rabbitmq_info' +# Override in config by specifying 'RmqcBin'. +RABBITMQCTL_BIN = '/usr/sbin/rabbitmqctl' +# Override in config by specifying 'PmapBin' +PMAP_BIN = '/usr/bin/pmap' +# Override in config by specifying 'PidofBin'. +PIDOF_BIN = '/bin/pidof' +# Override in config by specifying 'Verbose'. +VERBOSE_LOGGING = False + +# Wasn't specified for some reason... +PID_FILE = '/var/run/rabbitmq/pid' + + + +# Obtain the interesting statistical info +def get_stats(): + stats = {} + stats['ctl_messages'] = 0 + stats['ctl_memory'] = 0 + stats['ctl_consumers'] = 0 + stats['pmap_mapped'] = 0 + stats['pmap_used'] = 0 + stats['pmap_shared'] = 0 + + # call rabbitmqctl + try: + p = subprocess.Popen([RABBITMQCTL_BIN, '-q', 'list_queues', 'messages', 'memory', 'consumers'], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except: + logger('err', 'Failed to run %s' % RABBITMQCTL_BIN) + return None + + for line in p.stdout.readlines(): + if re.match('\d', line): + ctl_stats = line.split() + stats['ctl_messages'] += int(ctl_stats[0]) + stats['ctl_memory'] += int(ctl_stats[1]) + stats['ctl_consumers'] += int(ctl_stats[2]) + + if not stats['ctl_memory'] > 0: + logger('warn', '%s reports 0 memory usage. This is probably incorrect.' % RABBITMQCTL_BIN) + + # get the pid of rabbitmq + try: + with open(PID_FILE, 'r') as f: + pid = f.read().strip() + except: + logger('err', 'Unable to read %s' % PID_FILE) + return None + + # use pmap to get proper memory stats + try: + p = subprocess.Popen([PMAP_BIN, '-d', pid], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except: + logger('err', 'Failed to run %s' % PMAP_BIN) + return None + + line = p.stdout.readlines()[-1].strip() + if re.match('mapped', line): + m = re.match(r"\D+(\d+)\D+(\d+)\D+(\d+)", line) + stats['pmap_mapped'] = int(m.group(1)) + stats['pmap_used'] = int(m.group(2)) + stats['pmap_shared'] = int(m.group(3)) + else: + logger('warn', '%s returned something strange.' % PMAP_BIN) + return None + + # Verbose output + logger('verb', '[rmqctl] Messages: %i, Memory: %i, Consumers: %i' % (stats['ctl_messages'], stats['ctl_memory'], stats['ctl_consumers'])) + logger('verb', '[pmap] Mapped: %i, Used: %i, Shared: %i' % (stats['pmap_mapped'], stats['pmap_used'], stats['pmap_shared'])) + + return stats + + +# Config data from collectd +def configure_callback(conf): + global RABBITMQCTL_BIN, PMAP_BIN, PID_FILE, VERBOSE_LOGGING + for node in conf.children: + if node.key == 'RmqcBin': + RABBITMQCTL_BIN = node.values[0] + elif node.key == 'PmapBin': + PMAP_BIN = node.values[0] + elif node.key == 'PidFile': + PID_FILE = node.values[0] + elif node.key == 'Verbose': + VERBOSE_LOGGING = bool(node.values[0]) + else: + logger('warn', 'Unknown config key: %s' % node.key) + + +# Send info to collectd +def read_callback(): + logger('verb', 'read_callback') + info = get_stats() + + if not info: + logger('err', 'No information received - very bad.') + return + + logger('verb', 'About to trigger the dispatch..') + + # send values + for key in info: + logger('verb', 'Dispatching %s : %i' % (key, info[key])) + val = collectd.Values(plugin=NAME) + val.type = 'gauge' + val.type_instance = key + val.values = [int(info[key])] + val.dispatch() + + +# Send log messages (via collectd) +def logger(t, msg): + if t == 'err': + collectd.error('%s: %s' % (NAME, msg)) + if t == 'warn': + collectd.warning('%s: %s' % (NAME, msg)) + elif t == 'verb' and VERBOSE_LOGGING == True: + collectd.info('%s: %s' % (NAME, msg)) + + +# Runtime +collectd.register_config(configure_callback) +collectd.warning('Initialising rabbitmq_info') +collectd.register_read(read_callback) diff --git a/chef/cookbooks/collectd/metadata.rb b/chef/cookbooks/collectd/metadata.rb index af1a1ee..8c9a510 100644 --- a/chef/cookbooks/collectd/metadata.rb +++ b/chef/cookbooks/collectd/metadata.rb @@ -7,5 +7,4 @@ long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) version "1.0.3" supports "ubuntu" supports "centos" -depends "apt" depends "yum" diff --git a/chef/cookbooks/collectd/recipes/client.rb b/chef/cookbooks/collectd/recipes/client.rb index 3b132c3..3ab309c 100644 --- a/chef/cookbooks/collectd/recipes/client.rb +++ b/chef/cookbooks/collectd/recipes/client.rb @@ -2,7 +2,7 @@ # Cookbook Name:: collectd # Recipe:: client # -# Copyright 2010, Atari, Inc +# Copyright 2014, Huawei Technologies Co,ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,33 +18,17 @@ # include_recipe "collectd" -#servers = [] -#search(:node, 'recipes:collectd\\:\\:server') do |n| -# servers << n['fqdn'] -#end - -#if servers.empty? -# raise "No servers found. Please configure at least one node with collectd::server." -#end - -#collectd_plugin "network" do -# options :server=>servers -#end - -cookbook_file "#{node['collectd']['plugin_dir']}/kairosdb_writer.py" do - source "kairosdb_writer.py" - owner "root" - group "root" - mode 00644 - notifies :restart, "service[collectd]" - action :create_if_missing -end - -case node["platform_family"] -when "rhel" - node.override["collectd"]["plugins"]=node["collectd"]["rhel"]["plugins"].to_hash -when "debian" - node.override["collectd"]["plugins"]=node["collectd"]["debian"]["plugins"].to_hash +if node["collectd"].attribute?("rhel") or node["collectd"].attribute?("debian") + case node["platform_family"] + when "rhel" + if not node["collectd"]["rhel"]["plugins"].nil? + node.override["collectd"]["plugins"]=node["collectd"]["rhel"]["plugins"].to_hash + end + when "debian" + if not node["collectd"]["debian"]["plugins"].nil? + node.override["collectd"]["plugins"]=node["collectd"]["debian"]["plugins"].to_hash + end + end end node["collectd"]["plugins"].each_pair do |plugin_key, options| @@ -53,13 +37,9 @@ node["collectd"]["plugins"].each_pair do |plugin_key, options| end end -collectd_python_plugin "kairosdb_writer" do - opts = {"KairosDBHost"=>node['collectd']['server']['host'], - "KairosDBPort"=>node['collectd']['server']['port'], - "KairosDBProtocol"=>node['collectd']['server']['protocol'], - "LowercaseMetricNames"=>"true", - "Tags" => "host=#{node['fqdn']}\" \"role=OSROLE\" \"location=China.Beijing.TsingHua\" \"cluster=#{node['cluster']}", - "TypesDB" => node['collectd']['types_db'] - } - options(opts) +#for python plugins or more complicated ones, use seperate recipe to deploy them +if node["collectd"].attribute?("included_plugins") and not node["collectd"]["included_plugins"].nil? + node["collectd"]["included_plugins"].each_pair do |plugin_key, options| + include_recipe("collectd::#{plugin_key}") + end end diff --git a/chef/cookbooks/collectd/recipes/default.rb b/chef/cookbooks/collectd/recipes/default.rb index 7598f8d..f18dbda 100644 --- a/chef/cookbooks/collectd/recipes/default.rb +++ b/chef/cookbooks/collectd/recipes/default.rb @@ -17,10 +17,6 @@ # limitations under the License. # case node["platform_family"] -when "debian" - package "ubuntu-cloud-keyring" do - action :install - end when "rhel" include_recipe "yum::epel" execute "yum-update" do diff --git a/chef/cookbooks/collectd/recipes/kairosdb.rb b/chef/cookbooks/collectd/recipes/kairosdb.rb new file mode 100644 index 0000000..65031ae --- /dev/null +++ b/chef/cookbooks/collectd/recipes/kairosdb.rb @@ -0,0 +1,39 @@ +# +# Cookbook Name:: collectd +# Recipe:: kairosdb +# +# Copyright 2014, Huawei Technologies, Co,ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +cookbook_file "#{node['collectd']['plugin_dir']}/kairosdb_writer.py" do + source "kairosdb_writer.py" + owner "root" + group "root" + mode 00644 + action :create_if_missing +end + +if ! node['cluster'] + node.set['cluster'] = "no_cluster_defined" +end +collectd_python_plugin "kairosdb_writer" do + opts = {"KairosDBHost"=>node['collectd']['server']['host'], + "KairosDBPort"=>node['collectd']['server']['port'], + "KairosDBProtocol"=>node['collectd']['server']['protocol'], + "LowercaseMetricNames"=>"true", + "Tags" => "host=#{node['fqdn']}\" \"role=OSROLE\" \"location=China.Beijing.TsingHua\" \"cluster=#{node['cluster']}", + "TypesDB" => node['collectd']['types_db'] + } + options(opts) +end diff --git a/chef/cookbooks/collectd/recipes/rabbitmq.rb b/chef/cookbooks/collectd/recipes/rabbitmq.rb new file mode 100644 index 0000000..2ac08fb --- /dev/null +++ b/chef/cookbooks/collectd/recipes/rabbitmq.rb @@ -0,0 +1,28 @@ +# +# Cookbook Name:: collectd-plugins +# Recipe:: rabbitmq +# +# Copyright 2012, Rackspace Hosting, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +cookbook_file File.join(node['collectd']['plugin_dir'], "rabbitmq_info.py") do + source "rabbitmq_info.py" + owner "root" + group "root" + mode "0755" +end + +collectd_python_plugin "rabbitmq_info" diff --git a/chef/roles/os-ops-messaging.rb b/chef/roles/os-ops-messaging.rb index 0b679c5..2b5da61 100644 --- a/chef/roles/os-ops-messaging.rb +++ b/chef/roles/os-ops-messaging.rb @@ -14,7 +14,8 @@ override_attributes( "plugins" => { "processes" => { "Process" => ["rabbitmq-server"] } } - } + }, + "included_plugins" => {"rabbitmq"=>{}} } ) run_list(