From b4bafb06c05bfd5d88afc106f08611c1f5b6b107 Mon Sep 17 00:00:00 2001 From: Yanis Guenane Date: Mon, 13 Jan 2014 16:29:06 -0500 Subject: [PATCH 1/5] loadbalancer: Make it possible to define a master and a backup router Until now, the router was hardcoded to be MASTER in the keepalived configuration. This commit allows us to be able to define MASTER and BACKUPS routers for HA failover. close #25 --- manifests/loadbalancer.pp | 3 +- spec/classes/cloud_loadbalancer_spec.rb | 71 +++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/manifests/loadbalancer.pp b/manifests/loadbalancer.pp index 85a0c88b..906a6b43 100644 --- a/manifests/loadbalancer.pp +++ b/manifests/loadbalancer.pp @@ -33,6 +33,7 @@ class cloud::loadbalancer( $horizon = true, $spice = true, $haproxy_auth = $os_params::haproxy_auth, + $keepalived_state = 'BACKUP', $keepalived_interface = $os_params::keepalived_interface, $keepalived_ipvs = [ $os_params::openstack_vip, $os_params::mysql_vip ], $keepalived_localhost_ip = $os_params::keepalived_localhost_ip, @@ -66,7 +67,7 @@ class cloud::loadbalancer( keepalived::instance { '1': interface => $keepalived_interface, virtual_ips => split(join(flatten([$keepalived_ipvs, ['']]), " dev ${keepalived_interface},"), ','), - state => 'MASTER', + state => $keepalived_state, track_script => ['haproxy'], priority => 50, } diff --git a/spec/classes/cloud_loadbalancer_spec.rb b/spec/classes/cloud_loadbalancer_spec.rb index f8a17255..c2dfc322 100644 --- a/spec/classes/cloud_loadbalancer_spec.rb +++ b/spec/classes/cloud_loadbalancer_spec.rb @@ -20,7 +20,7 @@ require 'spec_helper' describe 'cloud::loadbalancer' do - shared_examples_for 'openstack loadbalancer' do + shared_examples_for 'openstack master loadbalancer' do let :params do { :ceilometer_api => true, @@ -39,6 +39,65 @@ describe 'cloud::loadbalancer' do :horizon => true, :spice => true, :haproxy_auth => 'root:secrete', + :keepalived_state => 'MASTER', + :keepalived_interface => 'eth0', + :keepalived_ipvs => ['10.0.0.1', '10.0.0.2'], + :keepalived_localhost_ip => '127.0.0.1', + :horizon_port => '80', + :spice_port => '6082', + :openstack_vip => '10.0.0.3', + :mysql_vip => '10.0.0.4', + :ks_ceilometer_public_port => '8777', + :ks_nova_public_port => '8774', + :ks_ec2_public_port => '8773', + :ks_metadata_public_port => '8777', + :ks_glance_public_port => '9292', + :ks_swift_public_port => '8080', + :ks_keystone_public_port => '5000', + :ks_keystone_admin_port => '35357', + :ks_cinder_public_port => '8776', + :ks_neutron_public_port => '9696', + :ks_heat_public_port => '8004', + :ks_heat_cfn_public_port => '8000', + :ks_heat_cloudwatch_public_port => '8003' } + end + + it 'configure haproxy server' do + should contain_class('haproxy') + end + + it 'configure keepalived server' do + should contain_class('keepalived') + end + + it 'configure vrrp_instance with MASTER state' do + should contain_keepalived__instance('1').with({ + 'state' => 'MASTER', + }) + end + + end + + shared_examples_for 'openstack backup loadbalancer' do + + let :params do + { :ceilometer_api => true, + :cinder_api => true, + :glance_api => true, + :neutron_api => true, + :heat_api => true, + :heat_cfn_api => true, + :heat_cloudwatch_api => true, + :nova_api => true, + :ec2_api => true, + :metadata_api => true, + :swift_api => true, + :keystone_api_admin => true, + :keystone_api => true, + :horizon => true, + :spice => true, + :haproxy_auth => 'root:secrete', + :keepalived_state => 'BACKUP', :keepalived_interface => 'eth0', :keepalived_ipvs => ['10.0.0.1', '10.0.0.2'], :keepalived_localhost_ip => '127.0.0.1', @@ -69,6 +128,10 @@ describe 'cloud::loadbalancer' do should contain_class('keepalived') end + it 'configure vrrp_instance with BACKUP state' do + should contain_keepalived__instance('1') + end + end context 'on Debian platforms' do @@ -77,7 +140,8 @@ describe 'cloud::loadbalancer' do :concat_basedir => '/var/lib/puppet/concat' } end - it_configures 'openstack loadbalancer' + it_configures 'openstack master loadbalancer' + it_configures 'openstack backup loadbalancer' end context 'on RedHat platforms' do @@ -86,7 +150,8 @@ describe 'cloud::loadbalancer' do :concat_basedir => '/var/lib/puppet/concat' } end - it_configures 'openstack loadbalancer' + it_configures 'openstack master loadbalancer' + it_configures 'openstack backup loadbalancer' end end From 183b99c0d276b8104ad055e24fa8581bf1c973fb Mon Sep 17 00:00:00 2001 From: Yanis Guenane Date: Mon, 13 Jan 2014 16:43:21 -0500 Subject: [PATCH 2/5] loadbalancer: Implementing a proper failover based on the router state Due to the fact that two HAProxy can not listen on the same vip at the same time, the management of the HAProxy service has been delegated to the keepalived process. With this patch, the keepalived process starts/stops the HAProxy daemon based on the state is is entering. If it enters MASTER mode then it starts the HAProxy daemon. If it enters BACKUP mode then it stops the HAProxy daemon. close #25 --- manifests/loadbalancer.pp | 16 ++++++++++------ spec/classes/cloud_loadbalancer_spec.rb | 17 +++++++++++++---- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/manifests/loadbalancer.pp b/manifests/loadbalancer.pp index 906a6b43..6d35d31b 100644 --- a/manifests/loadbalancer.pp +++ b/manifests/loadbalancer.pp @@ -56,7 +56,9 @@ class cloud::loadbalancer( $mysql_vip = $os_params::mysql_vip ){ - class { 'haproxy': } + class { 'haproxy': + manage_service => false, + } class { 'keepalived': } @@ -65,11 +67,13 @@ class cloud::loadbalancer( } keepalived::instance { '1': - interface => $keepalived_interface, - virtual_ips => split(join(flatten([$keepalived_ipvs, ['']]), " dev ${keepalived_interface},"), ','), - state => $keepalived_state, - track_script => ['haproxy'], - priority => 50, + interface => $keepalived_interface, + virtual_ips => split(join(flatten([$keepalived_ipvs, ['']]), " dev ${keepalived_interface},"), ','), + state => $keepalived_state, + track_script => ['haproxy'], + priority => 50, + notify_master => '"/etc/init.d/haproxy start"', + notify_backup => '"/etc/init.d/haproxy stop"', } $monitors_data = inline_template(' diff --git a/spec/classes/cloud_loadbalancer_spec.rb b/spec/classes/cloud_loadbalancer_spec.rb index c2dfc322..73a7de84 100644 --- a/spec/classes/cloud_loadbalancer_spec.rb +++ b/spec/classes/cloud_loadbalancer_spec.rb @@ -63,7 +63,9 @@ describe 'cloud::loadbalancer' do end it 'configure haproxy server' do - should contain_class('haproxy') + should contain_class('haproxy').with({ + 'manage_service' => 'false', + }) end it 'configure keepalived server' do @@ -72,7 +74,9 @@ describe 'cloud::loadbalancer' do it 'configure vrrp_instance with MASTER state' do should contain_keepalived__instance('1').with({ - 'state' => 'MASTER', + 'state' => 'MASTER', + 'notify_master' => '"/etc/init.d/haproxy start"', + 'notify_backup' => '"/etc/init.d/haproxy stop"', }) end @@ -121,7 +125,9 @@ describe 'cloud::loadbalancer' do end it 'configure haproxy server' do - should contain_class('haproxy') + should contain_class('haproxy').with({ + 'manage_service' => 'false', + }) end it 'configure keepalived server' do @@ -129,7 +135,10 @@ describe 'cloud::loadbalancer' do end it 'configure vrrp_instance with BACKUP state' do - should contain_keepalived__instance('1') + should contain_keepalived__instance('1').with({ + 'notify_master' => '"/etc/init.d/haproxy start"', + 'notify_backup' => '"/etc/init.d/haproxy stop"', + }) end end From 491f7bd30727c18fbf043d94b2293607e59633e9 Mon Sep 17 00:00:00 2001 From: Yanis Guenane Date: Tue, 14 Jan 2014 15:07:10 -0500 Subject: [PATCH 3/5] loadbalancer: Enable priority selection One can now define the priority of the vrrp router close #25 --- manifests/loadbalancer.pp | 3 ++- spec/classes/cloud_loadbalancer_spec.rb | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/manifests/loadbalancer.pp b/manifests/loadbalancer.pp index 6d35d31b..44356377 100644 --- a/manifests/loadbalancer.pp +++ b/manifests/loadbalancer.pp @@ -34,6 +34,7 @@ class cloud::loadbalancer( $spice = true, $haproxy_auth = $os_params::haproxy_auth, $keepalived_state = 'BACKUP', + $keepalived_priority = 50, $keepalived_interface = $os_params::keepalived_interface, $keepalived_ipvs = [ $os_params::openstack_vip, $os_params::mysql_vip ], $keepalived_localhost_ip = $os_params::keepalived_localhost_ip, @@ -71,7 +72,7 @@ class cloud::loadbalancer( virtual_ips => split(join(flatten([$keepalived_ipvs, ['']]), " dev ${keepalived_interface},"), ','), state => $keepalived_state, track_script => ['haproxy'], - priority => 50, + priority => $keepalived_priority, notify_master => '"/etc/init.d/haproxy start"', notify_backup => '"/etc/init.d/haproxy stop"', } diff --git a/spec/classes/cloud_loadbalancer_spec.rb b/spec/classes/cloud_loadbalancer_spec.rb index 73a7de84..6f1f4cb3 100644 --- a/spec/classes/cloud_loadbalancer_spec.rb +++ b/spec/classes/cloud_loadbalancer_spec.rb @@ -40,6 +40,7 @@ describe 'cloud::loadbalancer' do :spice => true, :haproxy_auth => 'root:secrete', :keepalived_state => 'MASTER', + :keepalived_priority => 50, :keepalived_interface => 'eth0', :keepalived_ipvs => ['10.0.0.1', '10.0.0.2'], :keepalived_localhost_ip => '127.0.0.1', @@ -75,6 +76,7 @@ describe 'cloud::loadbalancer' do it 'configure vrrp_instance with MASTER state' do should contain_keepalived__instance('1').with({ 'state' => 'MASTER', + 'priority' => 50, 'notify_master' => '"/etc/init.d/haproxy start"', 'notify_backup' => '"/etc/init.d/haproxy stop"', }) @@ -102,6 +104,7 @@ describe 'cloud::loadbalancer' do :spice => true, :haproxy_auth => 'root:secrete', :keepalived_state => 'BACKUP', + :keepalived_priority => 49, :keepalived_interface => 'eth0', :keepalived_ipvs => ['10.0.0.1', '10.0.0.2'], :keepalived_localhost_ip => '127.0.0.1', @@ -120,7 +123,7 @@ describe 'cloud::loadbalancer' do :ks_cinder_public_port => '8776', :ks_neutron_public_port => '9696', :ks_heat_public_port => '8004', - :ks_heat_cfn_public_port => '8000', + :ks_heat_cfn_public_port => '8000', :ks_heat_cloudwatch_public_port => '8003' } end @@ -136,6 +139,7 @@ describe 'cloud::loadbalancer' do it 'configure vrrp_instance with BACKUP state' do should contain_keepalived__instance('1').with({ + 'priority' => '49', 'notify_master' => '"/etc/init.d/haproxy start"', 'notify_backup' => '"/etc/init.d/haproxy stop"', }) From 6b772fc6a145607987a7b63aa1eb33e8d84d6c19 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Mon, 20 Jan 2014 16:46:16 +0100 Subject: [PATCH 4/5] loadbalancer: unit testing Signed-off-by: Emilien Macchi close #25 --- spec/classes/cloud_loadbalancer_spec.rb | 102 +++++++----------------- 1 file changed, 28 insertions(+), 74 deletions(-) diff --git a/spec/classes/cloud_loadbalancer_spec.rb b/spec/classes/cloud_loadbalancer_spec.rb index 6f1f4cb3..3a120ad3 100644 --- a/spec/classes/cloud_loadbalancer_spec.rb +++ b/spec/classes/cloud_loadbalancer_spec.rb @@ -20,7 +20,7 @@ require 'spec_helper' describe 'cloud::loadbalancer' do - shared_examples_for 'openstack master loadbalancer' do + shared_examples_for 'openstack loadbalancer' do let :params do { :ceilometer_api => true, @@ -39,7 +39,6 @@ describe 'cloud::loadbalancer' do :horizon => true, :spice => true, :haproxy_auth => 'root:secrete', - :keepalived_state => 'MASTER', :keepalived_priority => 50, :keepalived_interface => 'eth0', :keepalived_ipvs => ['10.0.0.1', '10.0.0.2'], @@ -73,76 +72,33 @@ describe 'cloud::loadbalancer' do should contain_class('keepalived') end - it 'configure vrrp_instance with MASTER state' do - should contain_keepalived__instance('1').with({ - 'state' => 'MASTER', - 'priority' => 50, - 'notify_master' => '"/etc/init.d/haproxy start"', - 'notify_backup' => '"/etc/init.d/haproxy stop"', - }) + context 'configure keepalived in backup' do + it 'configure vrrp_instance with BACKUP state' do + should contain_keepalived__instance('1').with({ + 'interface' => 'eth0', + 'track_script' => ['haproxy'], + 'state' => 'BACKUP', + 'priority' => 50, + 'notify_master' => '"/etc/init.d/haproxy start"', + 'notify_backup' => '"/etc/init.d/haproxy stop"', + }) + end end - end - - shared_examples_for 'openstack backup loadbalancer' do - - let :params do - { :ceilometer_api => true, - :cinder_api => true, - :glance_api => true, - :neutron_api => true, - :heat_api => true, - :heat_cfn_api => true, - :heat_cloudwatch_api => true, - :nova_api => true, - :ec2_api => true, - :metadata_api => true, - :swift_api => true, - :keystone_api_admin => true, - :keystone_api => true, - :horizon => true, - :spice => true, - :haproxy_auth => 'root:secrete', - :keepalived_state => 'BACKUP', - :keepalived_priority => 49, - :keepalived_interface => 'eth0', - :keepalived_ipvs => ['10.0.0.1', '10.0.0.2'], - :keepalived_localhost_ip => '127.0.0.1', - :horizon_port => '80', - :spice_port => '6082', - :openstack_vip => '10.0.0.3', - :mysql_vip => '10.0.0.4', - :ks_ceilometer_public_port => '8777', - :ks_nova_public_port => '8774', - :ks_ec2_public_port => '8773', - :ks_metadata_public_port => '8777', - :ks_glance_public_port => '9292', - :ks_swift_public_port => '8080', - :ks_keystone_public_port => '5000', - :ks_keystone_admin_port => '35357', - :ks_cinder_public_port => '8776', - :ks_neutron_public_port => '9696', - :ks_heat_public_port => '8004', - :ks_heat_cfn_public_port => '8000', - :ks_heat_cloudwatch_public_port => '8003' } - end - - it 'configure haproxy server' do - should contain_class('haproxy').with({ - 'manage_service' => 'false', - }) - end - - it 'configure keepalived server' do - should contain_class('keepalived') - end - - it 'configure vrrp_instance with BACKUP state' do - should contain_keepalived__instance('1').with({ - 'priority' => '49', - 'notify_master' => '"/etc/init.d/haproxy start"', - 'notify_backup' => '"/etc/init.d/haproxy stop"', - }) + context 'configure keepalived in master' do + before :each do + params.merge!( :keepalived_state => 'MASTER' ) + end + it 'configure vrrp_instance with MASTER state' do + should contain_keepalived__instance('1').with({ + 'interface' => 'eth0', + 'track_script' => ['haproxy'], + 'state' => 'MASTER', + 'priority' => 50, + 'notify_master' => '"/etc/init.d/haproxy start"', + 'notify_backup' => '"/etc/init.d/haproxy stop"', + }) + end end end @@ -153,8 +109,7 @@ describe 'cloud::loadbalancer' do :concat_basedir => '/var/lib/puppet/concat' } end - it_configures 'openstack master loadbalancer' - it_configures 'openstack backup loadbalancer' + it_configures 'openstack loadbalancer' end context 'on RedHat platforms' do @@ -163,8 +118,7 @@ describe 'cloud::loadbalancer' do :concat_basedir => '/var/lib/puppet/concat' } end - it_configures 'openstack master loadbalancer' - it_configures 'openstack backup loadbalancer' + it_configures 'openstack loadbalancer' end end From f75e7a1078485f6e0e714372762295f6bd8da5ab Mon Sep 17 00:00:00 2001 From: Sebastien Badia Date: Tue, 21 Jan 2014 23:44:45 +0100 Subject: [PATCH 5/5] Move inline_template to a real template (cleaner) Create a proper conf. file also for logrotate Refs: #25 --- files/logrotate/haproxy | 12 ++++ manifests/loadbalancer.pp | 88 +++--------------------------- templates/loadbalancer/monitor.erb | 60 ++++++++++++++++++++ 3 files changed, 80 insertions(+), 80 deletions(-) create mode 100644 files/logrotate/haproxy create mode 100644 templates/loadbalancer/monitor.erb diff --git a/files/logrotate/haproxy b/files/logrotate/haproxy new file mode 100644 index 00000000..0e2d7045 --- /dev/null +++ b/files/logrotate/haproxy @@ -0,0 +1,12 @@ +# Managed by Puppet +# Module cloud::loadbalancer +/var/log/haproxy.log { + rotate 7 + daily + missingok + notifempty + delaycompress + compress + postrotate + endscript +} diff --git a/manifests/loadbalancer.pp b/manifests/loadbalancer.pp index 44356377..987bbcde 100644 --- a/manifests/loadbalancer.pp +++ b/manifests/loadbalancer.pp @@ -15,7 +15,6 @@ # # HAproxy nodes # - class cloud::loadbalancer( $ceilometer_api = true, $cinder_api = true, @@ -36,7 +35,7 @@ class cloud::loadbalancer( $keepalived_state = 'BACKUP', $keepalived_priority = 50, $keepalived_interface = $os_params::keepalived_interface, - $keepalived_ipvs = [ $os_params::openstack_vip, $os_params::mysql_vip ], + $keepalived_ipvs = [$os_params::openstack_vip,$os_params::mysql_vip], $keepalived_localhost_ip = $os_params::keepalived_localhost_ip, $ks_cinder_public_port = $os_params::ks_cinder_public_port, $ks_ceilometer_public_port = $os_params::ks_ceilometer_public_port, @@ -77,83 +76,12 @@ class cloud::loadbalancer( notify_backup => '"/etc/init.d/haproxy stop"', } - $monitors_data = inline_template(' -<%- if @swift_api -%> -acl swift_api_dead nbsrv(swift_api_cluster) lt 1 -monitor fail if swift_api_dead -<%- end -%> -<%- if @keystone_api -%> -acl keystone_api_dead nbsrv(keystone_api_cluster) lt 1 -monitor fail if keystone_api_dead -<% end -%> -<%- if @galera -%> -acl galera_dead nbsrv(galera_cluster) lt 1 -monitor fail if galera_dead -<%- end -%> -<%- if @neutron_api -%> -acl neutron_api_dead nbsrv(neutron_api_cluster) lt 1 -monitor fail if neutron_api_dead -<%- end -%> -<%- if @cinder_api -%> -acl cinder_api_dead nbsrv(cinder_api_cluster) lt 1 -monitor fail if cinder_api_dead -<%- end -%> -<%- if @nova_api -%> -acl nova_api_dead nbsrv(nova_api_cluster) lt 1 -monitor fail if nova_api_dead -<%- end -%> -<%- if @nova_ec2 -%> -acl nova_ec2_dead nbsrv(nova_ec2_cluster) lt 1 -monitor fail if nova_ec2_dead -<%- end -%> -<%- if @nova_metadata -%> -acl nova_metadata_dead nbsrv(nova_metadata_cluster) lt 1 -monitor fail if nova_metadata_dead -<%- end -%> -<%- if @spice -%> -acl spice_dead nbsrv(spice_cluster) lt 1 -monitor fail if spice_dead -<%- end -%> -<%- if @glance_api -%> -acl nova_api_dead nbsrv(glance_api_cluster) lt 1 -monitor fail if nova_api_dead -<%- end -%> -<%- if @ceilometer_api -%> -acl ceilometer_api_dead nbsrv(ceilometer_api_cluster) lt 1 -monitor fail if ceilometer_api_dead -<%- end -%> -<%- if @heat_api -%> -acl heat_api_dead nbsrv(heat_api_cluster) lt 1 -monitor fail if heat_api_dead -<%- end -%> -<%- if @heat_cfn_api -%> -acl heat_api_cfn_dead nbsrv(heat_api_cfn_cluster) lt 1 -monitor fail if heat_api_cfn_dead -<%- end -%> -<%- if @heat_cloudwatch_api -%> -acl heat_api_cloudwatch_dead nbsrv(heat_api_cloudwatch_cluster) lt 1 -monitor fail if heat_api_cloudwatch_dead -<%- end -%> -<%- if @horizon -%> -acl horizon_dead nbsrv(horizon_cluster) lt 1 -monitor fail if horizon_dead -<%- end -%> -') - - file{'/etc/logrotate.d/haproxy': - content => " - /var/log/haproxy.log -{ - rotate 7 - daily - missingok - notifempty - delaycompress - compress - postrotate - endscript -} -" + file { '/etc/logrotate.d/haproxy': + ensure => file, + source => 'puppet:///cloud/logrotate/haproxy', + owner => root, + group => root, + mode => '0644'; } haproxy::listen { 'monitor': @@ -163,7 +91,7 @@ monitor fail if horizon_dead 'mode' => 'http', 'monitor-uri' => '/status', 'stats' => ['enable','uri /admin','realm Haproxy\ Statistics',"auth ${haproxy_auth}", 'refresh 5s' ], - '' => $monitors_data, + '' => template('cloud/loadbalancer/monitor.erb'), } } diff --git a/templates/loadbalancer/monitor.erb b/templates/loadbalancer/monitor.erb new file mode 100644 index 00000000..49aa679a --- /dev/null +++ b/templates/loadbalancer/monitor.erb @@ -0,0 +1,60 @@ +<%- if @swift_api -%> +acl swift_api_dead nbsrv(swift_api_cluster) lt 1 +monitor fail if swift_api_dead +<%- end -%> +<%- if @keystone_api -%> +acl keystone_api_dead nbsrv(keystone_api_cluster) lt 1 +monitor fail if keystone_api_dead +<% end -%> +<%- if @galera -%> +acl galera_dead nbsrv(galera_cluster) lt 1 +monitor fail if galera_dead +<%- end -%> +<%- if @neutron_api -%> +acl neutron_api_dead nbsrv(neutron_api_cluster) lt 1 +monitor fail if neutron_api_dead +<%- end -%> +<%- if @cinder_api -%> +acl cinder_api_dead nbsrv(cinder_api_cluster) lt 1 +monitor fail if cinder_api_dead +<%- end -%> +<%- if @nova_api -%> +acl nova_api_dead nbsrv(nova_api_cluster) lt 1 +monitor fail if nova_api_dead +<%- end -%> +<%- if @nova_ec2 -%> +acl nova_ec2_dead nbsrv(nova_ec2_cluster) lt 1 +monitor fail if nova_ec2_dead +<%- end -%> +<%- if @nova_metadata -%> +acl nova_metadata_dead nbsrv(nova_metadata_cluster) lt 1 +monitor fail if nova_metadata_dead +<%- end -%> +<%- if @spice -%> +acl spice_dead nbsrv(spice_cluster) lt 1 +monitor fail if spice_dead +<%- end -%> +<%- if @glance_api -%> +acl nova_api_dead nbsrv(glance_api_cluster) lt 1 +monitor fail if nova_api_dead +<%- end -%> +<%- if @ceilometer_api -%> +acl ceilometer_api_dead nbsrv(ceilometer_api_cluster) lt 1 +monitor fail if ceilometer_api_dead +<%- end -%> +<%- if @heat_api -%> +acl heat_api_dead nbsrv(heat_api_cluster) lt 1 +monitor fail if heat_api_dead +<%- end -%> +<%- if @heat_cfn_api -%> +acl heat_api_cfn_dead nbsrv(heat_api_cfn_cluster) lt 1 +monitor fail if heat_api_cfn_dead +<%- end -%> +<%- if @heat_cloudwatch_api -%> +acl heat_api_cloudwatch_dead nbsrv(heat_api_cloudwatch_cluster) lt 1 +monitor fail if heat_api_cloudwatch_dead +<%- end -%> +<%- if @horizon -%> +acl horizon_dead nbsrv(horizon_cluster) lt 1 +monitor fail if horizon_dead +<%- end -%>