From 9289d22180284d64ad258cd11cb2cc05d51dddd4 Mon Sep 17 00:00:00 2001 From: Yanis Guenane Date: Thu, 12 Feb 2015 08:21:17 -0500 Subject: [PATCH] Redis: Enable master failover This commit enable redis master failover. After a period of time a new master will be elected by the sentinel and the haproxy will route the stream to the correct redis node. Change-Id: Ib107fdf37e9d0a24f9be4ae73c1a1092507f84b6 --- manifests/database/nosql/redis/sentinel.pp | 25 +++- manifests/database/nosql/redis/server.pp | 13 ++ manifests/loadbalancer.pp | 43 ++++++- manifests/messaging.pp | 5 +- manifests/monitoring/server/sensu.pp | 121 +++++++----------- ...loud_database_nosql_redis_sentinel_spec.rb | 7 + templates/database/redis-notifications.sh.erb | 30 +++++ 7 files changed, 162 insertions(+), 82 deletions(-) create mode 100644 templates/database/redis-notifications.sh.erb diff --git a/manifests/database/nosql/redis/sentinel.pp b/manifests/database/nosql/redis/sentinel.pp index 9cd6395f..9ee65d43 100644 --- a/manifests/database/nosql/redis/sentinel.pp +++ b/manifests/database/nosql/redis/sentinel.pp @@ -24,18 +24,39 @@ # Used for firewall purpose. # Default to 26379 # +# [*haproxy_monitor_ip*] +# (optional) IP on which the HAProxy API is listening on +# Used for redis master failover purpose +# Default to 127.0.0.1 +# +# [*haproxy_monitor_port*] +# (optional) Port on which the HAProxy API is listening on +# Used for redis master failover purpose +# Default to 10300 +# # [*firewall_settings*] # (optional) Allow to add custom parameters to firewall rules # Should be an hash. # Default to {} # class cloud::database::nosql::redis::sentinel( - $port = 26379, - $firewall_settings = {}, + $port = 26379, + $haproxy_monitor_ip = '127.0.0.1', + $haproxy_monitor_port = '10300', + $firewall_settings = {}, ) { include ::redis::sentinel + file { '/bin/redis-notifications.sh': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + content => template('cloud/database/redis-notifications.sh.erb'), + before => Service['redis-sentinel'], + } + if $::cloud::manage_firewall { cloud::firewall::rule{ '100 allow redis sentinel access': port => $port, diff --git a/manifests/database/nosql/redis/server.pp b/manifests/database/nosql/redis/server.pp index 2a2ab9e3..cb627e7e 100644 --- a/manifests/database/nosql/redis/server.pp +++ b/manifests/database/nosql/redis/server.pp @@ -19,6 +19,10 @@ # # === Parameters: # +# [*bind_ip*] +# (optional) Address on which Redis is listening on +# Defaults to '127.0.0.1' +# # [*port*] # (optional) Port where Redis is binded. # Used for firewall purpose. @@ -30,12 +34,21 @@ # Default to {} # class cloud::database::nosql::redis::server( + $bind_ip = '127.0.0.1', $port = 6379, $firewall_settings = {}, ) { include ::redis + @@haproxy::balancermember{"${::fqdn}-redis": + listening_service => 'redis_cluster', + server_names => $::hostname, + ipaddresses => $bind_ip, + ports => $port, + options => 'check inter 2000 rise 2 fall 5' + } + if $::cloud::manage_firewall { cloud::firewall::rule{ '100 allow redis server access': port => $port, diff --git a/manifests/loadbalancer.pp b/manifests/loadbalancer.pp index f5d4e495..1c189202 100644 --- a/manifests/loadbalancer.pp +++ b/manifests/loadbalancer.pp @@ -178,6 +178,13 @@ # If set to false, no binding will be configure. # Defaults to true # +# [*redis*] +# (optional) Enable or not redis binding. +# If true, both public and internal will attempt to be created except if vip_internal_ip is set to false. +# If set to ['10.0.0.1'], only IP in the array (or in the string) will be configured in the pool. They must be part of keepalived_ip options. +# If set to false, no binding will be configure. +# Defaults to true +# # [*metadata_api*] # (optional) Enable or not Metadata public binding. # If true, both public and internal will attempt to be created except if vip_internal_ip is set to false. @@ -352,6 +359,11 @@ # service configuration block. # Defaults to [] # +# [*redis_bind_options*] +# (optional) A hash of options that are inserted into the HAproxy listening +# service configuration block. +# Defaults to [] +# # [*galera_bind_options*] # (optional) A hash of options that are inserted into the HAproxy listening # service configuration block. @@ -450,7 +462,11 @@ # # [*sensu_api_port*] # (optional) Port of Sensu API service. -# Defaults to '4567' +# Defaults to '4568' +# +# [*redis_port*] +# (optional) Port of redis service. +# Defaults to '6379' # # [*vip_public_ip*] # (optional) Array or string for public VIP @@ -504,6 +520,7 @@ class cloud::loadbalancer( $kibana = true, $sensu_dashboard = true, $sensu_api = true, + $redis = true, $haproxy_auth = 'admin:changeme', $keepalived_state = 'BACKUP', $keepalived_priority = '50', @@ -539,6 +556,7 @@ class cloud::loadbalancer( $kibana_bind_options = [], $sensu_dashboard_bind_options = [], $sensu_api_bind_options = [], + $redis_bind_options = [], $ks_ceilometer_public_port = 8777, $ks_cinder_public_port = 8776, $ks_ec2_public_port = 8773, @@ -562,7 +580,8 @@ class cloud::loadbalancer( $elasticsearch_port = 9200, $kibana_port = 8300, $sensu_dashboard_port = 3000, - $sensu_api_port = 4567, + $sensu_api_port = 4568, + $redis_port = 6379, $vip_public_ip = ['127.0.0.1'], $vip_internal_ip = false, $vip_monitor_ip = false, @@ -706,12 +725,19 @@ class cloud::loadbalancer( port => $sensu_dashboard_port, bind_options => $sensu_dashboard_bind_options, firewall_settings => $firewall_settings, + options => { + 'balance' => 'source', + }, } cloud::loadbalancer::binding { 'sensu_api': ip => $sensu_api, port => $sensu_api_port, bind_options => $sensu_api_bind_options, firewall_settings => $firewall_settings, + options => { + 'balance' => 'source', + 'rspadd' => ['Access-Control-Allow-Origin:\ *', 'Access-Control-Allow-Headers:\ origin,\ x-requested-with,\ content-type', 'Access-Control-Allow-Methods:\ PUT,\ GET,\ POST,\ DELETE,\ OPTIONS'], + }, } cloud::loadbalancer::binding { 'spice_cluster': ip => $spice, @@ -883,6 +909,19 @@ class cloud::loadbalancer( firewall_settings => $firewall_settings, } + cloud::loadbalancer::binding { 'redis_cluster': + ip => $redis, + port => $redis_port, + options => { + 'mode' => 'tcp', + 'balance' => 'first', + 'option' => ['tcp-check',], + 'tcp-check' => ['send info\ replication\r\n','expect string role:master'], + }, + bind_options => $redis_bind_options, + firewall_settings => $firewall_settings, + } + if (member(any2array($keepalived_public_ipvs), $galera_ip)) { warning('Exposing Galera cluster to public network is a security issue.') } diff --git a/manifests/messaging.pp b/manifests/messaging.pp index 9c86d0f4..ffb95ac7 100644 --- a/manifests/messaging.pp +++ b/manifests/messaging.pp @@ -102,11 +102,11 @@ class cloud::messaging( package_provider => $package_provider, } - rabbitmq_vhost { '/': + rabbitmq_vhost { ['/', '/sensu']: provider => 'rabbitmqctl', require => Class['rabbitmq'], } - rabbitmq_user { ['nova','glance','neutron','cinder','ceilometer','heat','trove']: + rabbitmq_user { ['nova','glance','neutron','cinder','ceilometer','heat','trove', 'sensu']: admin => true, password => $rabbit_password, provider => 'rabbitmqctl', @@ -120,6 +120,7 @@ class cloud::messaging( 'ceilometer@/', 'heat@/', 'trove@/', + 'sensu@/sensu', ]: configure_permission => '.*', write_permission => '.*', diff --git a/manifests/monitoring/server/sensu.pp b/manifests/monitoring/server/sensu.pp index 5c3828b0..d2484534 100644 --- a/manifests/monitoring/server/sensu.pp +++ b/manifests/monitoring/server/sensu.pp @@ -44,30 +44,18 @@ # } # } # -# [*manage_rabbitmq_resources*] -# (optionnal) A boolean that determines if the RabbitMQ resources should be exported +# [*manage_sensu_plugins*] +# (optionnal) A boolean that determines if the Sensu plugins resources should be exported # from this node # Defaults to 'true' # -# [*rabbitmq_user*] -# (optionnal) Rabbitmq user -# Defaults to 'sensu' -# -# [*rabbitmq_password*] -# (optionnal) Rabbitmq_password -# Defaults to 'rabbitpassword' -# -# [*rabbitmq_vhost*] -# (optionnal) Rabbitmq vhost -# Defaults to '/sensu' -# # [*sensu_api_ip*] # (optionnal) IP address to bind the sensu_api to # Defaults to '%{::ipaddress}' # # [*sensu_api_port*] # (optionnal) Port to bind the sensu_api to -# Defaults to '4567' +# Defaults to '4568' # # [*uchiwa_ip*] # (optionnal) IP address to bind uchiwa to @@ -86,12 +74,9 @@ class cloud::monitoring::server::sensu ( $checks = {}, $handlers = {}, $plugins = {}, - $manage_rabbitmq_resources = true, - $rabbitmq_user = 'sensu', - $rabbitmq_password = 'rabbitpassword', - $rabbitmq_vhost = '/sensu', + $manage_sensu_plugins = true, $sensu_api_ip = $::ipaddress, - $sensu_api_port = '4567', + $sensu_api_port = '4568', $uchiwa_ip = $::ipaddress, $uchiwa_port = '3000', $firewall_settings = {}, @@ -99,67 +84,51 @@ class cloud::monitoring::server::sensu ( include cloud::params - if $manage_rabbitmq_resources { - @@rabbitmq_user { $rabbitmq_user : - password => $rabbitmq_password, + Service['sensu-api'] -> Service['uchiwa'] + Service['sensu-server'] -> Service['uchiwa'] + Service['sensu-server'] -> Sensu::Plugin <<| |>> + + include cloud::monitoring::agent::sensu + + create_resources('sensu::check', $checks) + create_resources('sensu::handler', $handlers) + + if $manage_sensu_plugins { + create_resources('@@sensu::plugin', $plugins) + } + + include ::uchiwa + uchiwa::api { 'OpenStack' : + host => $sensu_api_ip, + port => $sensu_api_port, + } + + if $::cloud::manage_firewall { + cloud::firewall::rule{ '100 allow sensu_dashboard access': + port => $uchiwa_port, + extras => $firewall_settings, } - @@rabbitmq_vhost { $rabbitmq_vhost : - ensure => present, - } - @@rabbitmq_user_permissions { "${rabbitmq_user}@${rabbitmq_vhost}" : - configure_permission => '.*', - read_permission => '.*', - write_permission => '.*', + + cloud::firewall::rule{ '100 allow sensu_api access': + port => $sensu_api_port, + extras => $firewall_settings, } } - $rabbitmq_user_realized = query_nodes("Rabbitmq_user['${rabbitmq_user}']") + @@haproxy::balancermember{"${::fqdn}-sensu_dashboard": + listening_service => 'sensu_dashboard', + server_names => $::hostname, + ipaddresses => $uchiwa_ip, + ports => $uchiwa_port, + options => 'check inter 2000 rise 2 fall 5' + } - if size($rabbitmq_user_realized) >= 1 { - - Service["${::cloud::params::redis_service_name}"] -> Service['sensu-api'] -> Service['sensu-server'] -> Service['uchiwa'] - Service['sensu-server'] -> Sensu::Plugin <<| |>> - - - include cloud::monitoring::agent::sensu - - create_resources('sensu::check', $checks) - create_resources('sensu::handler', $handlers) - create_resources('@@sensu::plugin', $plugins) - - include ::uchiwa - uchiwa::api { 'OpenStack' : - host => $uchiwa_ip, - port => $uchiwa_port, - } - - if $::cloud::manage_firewall { - cloud::firewall::rule{ '100 allow sensu_dashboard access': - port => $uchiwa_port, - extras => $firewall_settings, - } - - cloud::firewall::rule{ '100 allow sensu_api access': - port => $sensu_api_port, - extras => $firewall_settings, - } - } - - @@haproxy::balancermember{"${::fqdn}-sensu_dashboard": - listening_service => 'sensu_dashoard', - server_names => $::hostname, - ipaddresses => $uchiwa_ip, - ports => $uchiwa_port, - options => 'check inter 2000 rise 2 fall 5' - } - - @@haproxy::balancermember{"${::fqdn}-sensu_api": - listening_service => 'sensu_api', - server_names => $::hostname, - ipaddresses => $sensu_api_ip, - ports => $sensu_api_port, - options => 'check inter 2000 rise 2 fall 5' - } + @@haproxy::balancermember{"${::fqdn}-sensu_api": + listening_service => 'sensu_api', + server_names => $::hostname, + ipaddresses => $sensu_api_ip, + ports => $sensu_api_port, + options => 'check inter 2000 rise 2 fall 5' } } diff --git a/spec/classes/cloud_database_nosql_redis_sentinel_spec.rb b/spec/classes/cloud_database_nosql_redis_sentinel_spec.rb index df02503e..a7dbc69b 100644 --- a/spec/classes/cloud_database_nosql_redis_sentinel_spec.rb +++ b/spec/classes/cloud_database_nosql_redis_sentinel_spec.rb @@ -28,6 +28,13 @@ describe 'cloud::database::nosql::redis::sentinel' do it { should create_class('redis::sentinel') } + it { should contain_file('/bin/redis-notifications.sh').with( + :ensure => 'present', + :owner => 'root', + :group => 'root', + :mode => '0755', + ) } + context 'with default firewall enabled' do let :pre_condition do "class { 'cloud': manage_firewall => true }" diff --git a/templates/database/redis-notifications.sh.erb b/templates/database/redis-notifications.sh.erb new file mode 100644 index 00000000..fc5d9474 --- /dev/null +++ b/templates/database/redis-notifications.sh.erb @@ -0,0 +1,30 @@ +#!/bin/bash +HAPROXY="<%= @haproxy_monitor_ip %>:<%= @haproxy_monitor_port %>" +CMD="$1" +ARGS="$2" +ARG1=`echo $ARGS | awk '{print $1}'` + + +call_curl () { + DATA=`echo "s=$1&action=$2&b=%234" | sed -e s/:/%3A/` + curl --silent -o /dev/null $HAPROXY --data "$DATA" + echo curl $HAPROXY --data "$DATA" + return 0 +} + + +[ "$CMD" = "+odown" ] && [ "$ARG1" = "master" ] && \ + call_curl `echo $ARGS | awk '{print $2 ":" $3 ":" $4}'` 'disable' + +[ "$CMD" = "+sdown" ] && [ "$ARG1" = "slave" ] && \ + call_curl `echo $ARGS | awk '{print $6 ":" $3 ":" $4}'` 'disable' + +[ "$CMD" = "+switch-master" ] && \ + call_curl `echo $ARGS | awk '{print $1 ":" $4 ":" $5}'` 'enable' && + call_curl `echo $ARGS | awk '{print $1 ":" $2 ":" $3}'` 'disable' + +[ "$CMD" = "-odown" ] && [ "$ARG1" = "master" ] && \ + call_curl `echo $ARGS | awk '{print $2 ":" $3 ":" $4}'` 'enable' + +# without exit code, sentinel thinks the script is still running and locks any further execution +exit 0