stackforge/puppet-openstack-cloud
Code Issues Proposed changes

Redis: Enable master failover

Browse Source 
This commit enable redis master failover. After a period of time
a new master will be elected by the sentinel and the haproxy will route
the stream to the correct redis node.

Change-Id: Ib107fdf37e9d0a24f9be4ae73c1a1092507f84b6
This commit is contained in:
Yanis Guenane 2015-02-12 08:21:17 -05:00
parent 5e76243b08
commit 9289d22180
7 changed files with 162 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
manifests/database/nosql/redis/sentinel.pp
View File

@ -24,18 +24,39 @@
# Used for firewall purpose.
# Default to 26379
#
# [*haproxy_monitor_ip*]
# (optional) IP on which the HAProxy API is listening on
# Used for redis master failover purpose
# Default to 127.0.0.1
#
# [*haproxy_monitor_port*]
# (optional) Port on which the HAProxy API is listening on
# Used for redis master failover purpose
# Default to 10300
#
# [*firewall_settings*]
# (optional) Allow to add custom parameters to firewall rules
# Should be an hash.
# Default to {}
#
class cloud::database::nosql::redis::sentinel(
$port = 26379,
$firewall_settings = {},
$port = 26379,
$haproxy_monitor_ip = '127.0.0.1',
$haproxy_monitor_port = '10300',
$firewall_settings = {},
) {
include ::redis::sentinel
file { '/bin/redis-notifications.sh':
ensure => present,
owner => 'root',
group => 'root',
mode => '0755',
content => template('cloud/database/redis-notifications.sh.erb'),
before => Service['redis-sentinel'],
}
if $::cloud::manage_firewall {
cloud::firewall::rule{ '100 allow redis sentinel access':
port => $port,

13
manifests/database/nosql/redis/server.pp
View File

@ -19,6 +19,10 @@
#
# === Parameters:
#
# [*bind_ip*]
# (optional) Address on which Redis is listening on
# Defaults to '127.0.0.1'
#
# [*port*]
# (optional) Port where Redis is binded.
# Used for firewall purpose.
@ -30,12 +34,21 @@
# Default to {}
#
class cloud::database::nosql::redis::server(
$bind_ip = '127.0.0.1',
$port = 6379,
$firewall_settings = {},
) {
include ::redis
@@haproxy::balancermember{"${::fqdn}-redis":
listening_service => 'redis_cluster',
server_names => $::hostname,
ipaddresses => $bind_ip,
ports => $port,
options => 'check inter 2000 rise 2 fall 5'
}
if $::cloud::manage_firewall {
cloud::firewall::rule{ '100 allow redis server access':
port => $port,

43
manifests/loadbalancer.pp
View File

@ -178,6 +178,13 @@
# If set to false, no binding will be configure.
# Defaults to true
#
# [*redis*]
# (optional) Enable or not redis binding.
# If true, both public and internal will attempt to be created except if vip_internal_ip is set to false.
# If set to ['10.0.0.1'], only IP in the array (or in the string) will be configured in the pool. They must be part of keepalived_ip options.
# If set to false, no binding will be configure.
# Defaults to true
#
# [*metadata_api*]
# (optional) Enable or not Metadata public binding.
# If true, both public and internal will attempt to be created except if vip_internal_ip is set to false.
@ -352,6 +359,11 @@
# service configuration block.
# Defaults to []
#
# [*redis_bind_options*]
# (optional) A hash of options that are inserted into the HAproxy listening
# service configuration block.
# Defaults to []
#
# [*galera_bind_options*]
# (optional) A hash of options that are inserted into the HAproxy listening
# service configuration block.
@ -450,7 +462,11 @@
#
# [*sensu_api_port*]
# (optional) Port of Sensu API service.
# Defaults to '4567'
# Defaults to '4568'
#
# [*redis_port*]
# (optional) Port of redis service.
# Defaults to '6379'
#
# [*vip_public_ip*]
# (optional) Array or string for public VIP
@ -504,6 +520,7 @@ class cloud::loadbalancer(
$kibana = true,
$sensu_dashboard = true,
$sensu_api = true,
$redis = true,
$haproxy_auth = 'admin:changeme',
$keepalived_state = 'BACKUP',
$keepalived_priority = '50',
@ -539,6 +556,7 @@ class cloud::loadbalancer(
$kibana_bind_options = [],
$sensu_dashboard_bind_options = [],
$sensu_api_bind_options = [],
$redis_bind_options = [],
$ks_ceilometer_public_port = 8777,
$ks_cinder_public_port = 8776,
$ks_ec2_public_port = 8773,
@ -562,7 +580,8 @@ class cloud::loadbalancer(
$elasticsearch_port = 9200,
$kibana_port = 8300,
$sensu_dashboard_port = 3000,
$sensu_api_port = 4567,
$sensu_api_port = 4568,
$redis_port = 6379,
$vip_public_ip = ['127.0.0.1'],
$vip_internal_ip = false,
$vip_monitor_ip = false,
@ -706,12 +725,19 @@ class cloud::loadbalancer(
port => $sensu_dashboard_port,
bind_options => $sensu_dashboard_bind_options,
firewall_settings => $firewall_settings,
options => {
'balance' => 'source',
},
}
cloud::loadbalancer::binding { 'sensu_api':
ip => $sensu_api,
port => $sensu_api_port,
bind_options => $sensu_api_bind_options,
firewall_settings => $firewall_settings,
options => {
'balance' => 'source',
'rspadd' => ['Access-Control-Allow-Origin:\ *', 'Access-Control-Allow-Headers:\ origin,\ x-requested-with,\ content-type', 'Access-Control-Allow-Methods:\ PUT,\ GET,\ POST,\ DELETE,\ OPTIONS'],
},
}
cloud::loadbalancer::binding { 'spice_cluster':
ip => $spice,
@ -883,6 +909,19 @@ class cloud::loadbalancer(
firewall_settings => $firewall_settings,
}
cloud::loadbalancer::binding { 'redis_cluster':
ip => $redis,
port => $redis_port,
options => {
'mode' => 'tcp',
'balance' => 'first',
'option' => ['tcp-check',],
'tcp-check' => ['send info\ replication\r\n','expect string role:master'],
},
bind_options => $redis_bind_options,
firewall_settings => $firewall_settings,
}
if (member(any2array($keepalived_public_ipvs), $galera_ip)) {
warning('Exposing Galera cluster to public network is a security issue.')
}

5
manifests/messaging.pp
View File

@ -102,11 +102,11 @@ class cloud::messaging(
package_provider => $package_provider,
}
rabbitmq_vhost { '/':
rabbitmq_vhost { ['/', '/sensu']:
provider => 'rabbitmqctl',
require => Class['rabbitmq'],
}
rabbitmq_user { ['nova','glance','neutron','cinder','ceilometer','heat','trove']:
rabbitmq_user { ['nova','glance','neutron','cinder','ceilometer','heat','trove', 'sensu']:
admin => true,
password => $rabbit_password,
provider => 'rabbitmqctl',
@ -120,6 +120,7 @@ class cloud::messaging(
'ceilometer@/',
'heat@/',
'trove@/',
'sensu@/sensu',
]:
configure_permission => '.*',
write_permission => '.*',

121
manifests/monitoring/server/sensu.pp
View File

@ -44,30 +44,18 @@
# }
# }
#
# [*manage_rabbitmq_resources*]
# (optionnal) A boolean that determines if the RabbitMQ resources should be exported
# [*manage_sensu_plugins*]
# (optionnal) A boolean that determines if the Sensu plugins resources should be exported
# from this node
# Defaults to 'true'
#
# [*rabbitmq_user*]
# (optionnal) Rabbitmq user
# Defaults to 'sensu'
#
# [*rabbitmq_password*]
# (optionnal) Rabbitmq_password
# Defaults to 'rabbitpassword'
#
# [*rabbitmq_vhost*]
# (optionnal) Rabbitmq vhost
# Defaults to '/sensu'
#
# [*sensu_api_ip*]
# (optionnal) IP address to bind the sensu_api to
# Defaults to '%{::ipaddress}'
#
# [*sensu_api_port*]
# (optionnal) Port to bind the sensu_api to
# Defaults to '4567'
# Defaults to '4568'
#
# [*uchiwa_ip*]
# (optionnal) IP address to bind uchiwa to
@ -86,12 +74,9 @@ class cloud::monitoring::server::sensu (
$checks = {},
$handlers = {},
$plugins = {},
$manage_rabbitmq_resources = true,
$rabbitmq_user = 'sensu',
$rabbitmq_password = 'rabbitpassword',
$rabbitmq_vhost = '/sensu',
$manage_sensu_plugins = true,
$sensu_api_ip = $::ipaddress,
$sensu_api_port = '4567',
$sensu_api_port = '4568',
$uchiwa_ip = $::ipaddress,
$uchiwa_port = '3000',
$firewall_settings = {},
@ -99,67 +84,51 @@ class cloud::monitoring::server::sensu (
include cloud::params
if $manage_rabbitmq_resources {
@@rabbitmq_user { $rabbitmq_user :
password => $rabbitmq_password,
Service['sensu-api'] -> Service['uchiwa']
Service['sensu-server'] -> Service['uchiwa']
Service['sensu-server'] -> Sensu::Plugin <<| |>>
include cloud::monitoring::agent::sensu
create_resources('sensu::check', $checks)
create_resources('sensu::handler', $handlers)
if $manage_sensu_plugins {
create_resources('@@sensu::plugin', $plugins)
}
include ::uchiwa
uchiwa::api { 'OpenStack' :
host => $sensu_api_ip,
port => $sensu_api_port,
}
if $::cloud::manage_firewall {
cloud::firewall::rule{ '100 allow sensu_dashboard access':
port => $uchiwa_port,
extras => $firewall_settings,
}
@@rabbitmq_vhost { $rabbitmq_vhost :
ensure => present,
}
@@rabbitmq_user_permissions { "${rabbitmq_user}@${rabbitmq_vhost}" :
configure_permission => '.*',
read_permission => '.*',
write_permission => '.*',
cloud::firewall::rule{ '100 allow sensu_api access':
port => $sensu_api_port,
extras => $firewall_settings,
}
}
$rabbitmq_user_realized = query_nodes("Rabbitmq_user['${rabbitmq_user}']")
@@haproxy::balancermember{"${::fqdn}-sensu_dashboard":
listening_service => 'sensu_dashboard',
server_names => $::hostname,
ipaddresses => $uchiwa_ip,
ports => $uchiwa_port,
options => 'check inter 2000 rise 2 fall 5'
}
if size($rabbitmq_user_realized) >= 1 {
Service["${::cloud::params::redis_service_name}"] -> Service['sensu-api'] -> Service['sensu-server'] -> Service['uchiwa']
Service['sensu-server'] -> Sensu::Plugin <<| |>>
include cloud::monitoring::agent::sensu
create_resources('sensu::check', $checks)
create_resources('sensu::handler', $handlers)
create_resources('@@sensu::plugin', $plugins)
include ::uchiwa
uchiwa::api { 'OpenStack' :
host => $uchiwa_ip,
port => $uchiwa_port,
}
if $::cloud::manage_firewall {
cloud::firewall::rule{ '100 allow sensu_dashboard access':
port => $uchiwa_port,
extras => $firewall_settings,
}
cloud::firewall::rule{ '100 allow sensu_api access':
port => $sensu_api_port,
extras => $firewall_settings,
}
}
@@haproxy::balancermember{"${::fqdn}-sensu_dashboard":
listening_service => 'sensu_dashoard',
server_names => $::hostname,
ipaddresses => $uchiwa_ip,
ports => $uchiwa_port,
options => 'check inter 2000 rise 2 fall 5'
}
@@haproxy::balancermember{"${::fqdn}-sensu_api":
listening_service => 'sensu_api',
server_names => $::hostname,
ipaddresses => $sensu_api_ip,
ports => $sensu_api_port,
options => 'check inter 2000 rise 2 fall 5'
}
@@haproxy::balancermember{"${::fqdn}-sensu_api":
listening_service => 'sensu_api',
server_names => $::hostname,
ipaddresses => $sensu_api_ip,
ports => $sensu_api_port,
options => 'check inter 2000 rise 2 fall 5'
}
}

7
spec/classes/cloud_database_nosql_redis_sentinel_spec.rb
View File

@ -28,6 +28,13 @@ describe 'cloud::database::nosql::redis::sentinel' do
it { should create_class('redis::sentinel') }
it { should contain_file('/bin/redis-notifications.sh').with(
:ensure => 'present',
:owner => 'root',
:group => 'root',
:mode => '0755',
) }
context 'with default firewall enabled' do
let :pre_condition do
"class { 'cloud': manage_firewall => true }"

30
templates/database/redis-notifications.sh.erb Normal file
View File

@ -0,0 +1,30 @@
#!/bin/bash
HAPROXY="<%= @haproxy_monitor_ip %>:<%= @haproxy_monitor_port %>"
CMD="$1"
ARGS="$2"
ARG1=`echo $ARGS | awk '{print $1}'`
call_curl () {
DATA=`echo "s=$1&action=$2&b=%234" | sed -e s/:/%3A/`
curl --silent -o /dev/null $HAPROXY --data "$DATA"
echo curl $HAPROXY --data "$DATA"
return 0
}
[ "$CMD" = "+odown" ] && [ "$ARG1" = "master" ] && \
call_curl `echo $ARGS | awk '{print $2 ":" $3 ":" $4}'` 'disable'
[ "$CMD" = "+sdown" ] && [ "$ARG1" = "slave" ] && \
call_curl `echo $ARGS | awk '{print $6 ":" $3 ":" $4}'` 'disable'
[ "$CMD" = "+switch-master" ] && \
call_curl `echo $ARGS | awk '{print $1 ":" $4 ":" $5}'` 'enable' &&
call_curl `echo $ARGS | awk '{print $1 ":" $2 ":" $3}'` 'disable'
[ "$CMD" = "-odown" ] && [ "$ARG1" = "master" ] && \
call_curl `echo $ARGS | awk '{print $2 ":" $3 ":" $4}'` 'enable'
# without exit code, sentinel thinks the script is still running and locks any further execution
exit 0