From a8e71e7b338663ed0f833019e108f0d85292065c Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Thu, 9 Apr 2015 10:19:56 -0400 Subject: [PATCH] loadbalancer: use TCP checks for API services While we don't have HTTP healthcheck in OpenStack [1], let's use TCP-connection level to validate a service is up and running. The reason is when "option httpchk" is specified, a complete HTTP request is sent once the TCP connection is established, and responses 2xx and 3xx are considered valid. So given that, OpenStack APIs are responding with 401 or 500, this healthcheck appears to be self-defeating [2]. Also introduce a new parameter to allow flexible timeout: api_timeout (beside galera_timeout). Sources: [1] http://specs.openstack.org/openstack/oslo-specs/specs/kilo/oslo-middleware-healthcheck.html [2] http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-option%20httpchk Change-Id: I0121d28daafa2f509162880e9934eb3e08ae4543 --- manifests/loadbalancer.pp | 86 ++++++++++++++----------- spec/classes/cloud_loadbalancer_spec.rb | 37 ++++++----- 2 files changed, 69 insertions(+), 54 deletions(-) diff --git a/manifests/loadbalancer.pp b/manifests/loadbalancer.pp index f4fe36c1..0a922e57 100644 --- a/manifests/loadbalancer.pp +++ b/manifests/loadbalancer.pp @@ -485,6 +485,10 @@ # *_db_idle_timeout for all services to be a little less # than this timeout. # +# [*api_timeout*] +# (optional) Timeout for API services connections +# Defaults to '90m'. +# # [*vip_public_ip*] # (optional) Array or string for public VIP # Should be part of keepalived_public_ips @@ -602,6 +606,7 @@ class cloud::loadbalancer( $sensu_api_port = 4568, $redis_port = 6379, $galera_timeout = '90m', + $api_timeout = '90m', $vip_public_ip = ['127.0.0.1'], $vip_internal_ip = false, $vip_monitor_ip = false, @@ -612,6 +617,14 @@ class cloud::loadbalancer( include cloud::params + $common_tcp_options = { + 'mode' => 'tcp', + 'option' => ['tcpka', 'tcplog', 'forwardfor'], + 'balance' => 'source', + 'timeout server' => $api_timeout, + 'timeout client' => $api_timeout, + } + if $keepalived_vrrp_interface { $keepalived_vrrp_interface_real = $keepalived_vrrp_interface } else { @@ -706,12 +719,14 @@ class cloud::loadbalancer( cloud::loadbalancer::binding { 'keystone_api_cluster': ip => $keystone_api, port => $ks_keystone_public_port, + options => $common_tcp_options, bind_options => $keystone_bind_options, firewall_settings => $firewall_settings, } cloud::loadbalancer::binding { 'keystone_api_admin_cluster': ip => $keystone_api_admin, port => $ks_keystone_admin_port, + options => $common_tcp_options, bind_options => $keystone_admin_bind_options, firewall_settings => $firewall_settings, } @@ -725,18 +740,21 @@ class cloud::loadbalancer( cloud::loadbalancer::binding { 'nova_api_cluster': ip => $nova_api, port => $ks_nova_public_port, + options => $common_tcp_options, bind_options => $nova_bind_options, firewall_settings => $firewall_settings, } cloud::loadbalancer::binding { 'ec2_api_cluster': ip => $ec2_api, port => $ks_ec2_public_port, + options => $common_tcp_options, bind_options => $ec2_bind_options, firewall_settings => $firewall_settings, } cloud::loadbalancer::binding { 'metadata_api_cluster': ip => $metadata_api, port => $ks_metadata_public_port, + options => $common_tcp_options, bind_options => $metadata_bind_options, firewall_settings => $firewall_settings, } @@ -762,26 +780,14 @@ class cloud::loadbalancer( cloud::loadbalancer::binding { 'spice_cluster': ip => $spice, port => $spice_port, - options => { - 'mode' => 'tcp', - 'option' => ['tcpka', 'tcplog', 'forwardfor'], - 'balance' => 'source', - 'timeout server' => '120m', - 'timeout client' => '120m', - }, + options => $common_tcp_options, bind_options => $spice_bind_options, firewall_settings => $firewall_settings, } cloud::loadbalancer::binding { 'novnc_cluster': ip => $novnc, port => $novnc_port, - options => { - 'mode' => 'tcp', - 'option' => ['tcpka', 'tcplog', 'forwardfor'], - 'balance' => 'source', - 'timeout server' => '120m', - 'timeout client' => '120m', - }, + options => $common_tcp_options, bind_options => $novnc_bind_options, firewall_settings => $firewall_settings, } @@ -789,9 +795,11 @@ class cloud::loadbalancer( ip => $rabbitmq, port => $rabbitmq_port, options => { - 'mode' => 'tcp', - 'option' => ['tcpka', 'tcplog', 'forwardfor'], - 'balance' => 'roundrobin', + 'mode' => 'tcp', + 'option' => ['tcpka', 'tcplog', 'forwardfor'], + 'balance' => 'roundrobin', + 'timeout server' => $api_timeout, + 'timeout client' => $api_timeout, }, bind_options => $rabbitmq_bind_options, firewall_settings => $firewall_settings, @@ -799,18 +807,13 @@ class cloud::loadbalancer( cloud::loadbalancer::binding { 'trove_api_cluster': ip => $trove_api, port => $ks_trove_public_port, + options => $common_tcp_options, bind_options => $trove_bind_options, firewall_settings => $firewall_settings, } cloud::loadbalancer::binding { 'glance_api_cluster': ip => $glance_api, - options => { - 'mode' => 'tcp', - 'balance' => 'source', - 'option' => ['tcpka', 'tcplog', 'forwardfor'], - 'timeout server' => '120m', - 'timeout client' => '120m', - }, + options => $common_tcp_options, port => $ks_glance_api_public_port, bind_options => $glance_api_bind_options, firewall_settings => $firewall_settings, @@ -818,45 +821,51 @@ class cloud::loadbalancer( cloud::loadbalancer::binding { 'glance_registry_cluster': ip => $glance_registry, port => $ks_glance_registry_internal_port, + options => $common_tcp_options, bind_options => $glance_registry_bind_options, firewall_settings => $firewall_settings, } cloud::loadbalancer::binding { 'neutron_api_cluster': ip => $neutron_api, port => $ks_neutron_public_port, + options => $common_tcp_options, bind_options => $neutron_bind_options, firewall_settings => $firewall_settings, } cloud::loadbalancer::binding { 'cinder_api_cluster': ip => $cinder_api, port => $ks_cinder_public_port, + options => $common_tcp_options, bind_options => $cinder_bind_options, firewall_settings => $firewall_settings, } cloud::loadbalancer::binding { 'ceilometer_api_cluster': ip => $ceilometer_api, port => $ks_ceilometer_public_port, + options => $common_tcp_options, bind_options => $ceilometer_bind_options, firewall_settings => $firewall_settings, } if 'ssl' in $heat_api_bind_options { - $heat_api_options = { - 'reqadd' => 'X-Forwarded-Proto:\ https if { ssl_fc }' } + $heat_api_options = merge($common_tcp_options, { + 'reqadd' => 'X-Forwarded-Proto:\ https if { ssl_fc }', + }) } else { - $heat_api_options = {} + $heat_api_options = $common_tcp_options } cloud::loadbalancer::binding { 'heat_api_cluster': ip => $heat_api, port => $ks_heat_public_port, - bind_options => $heat_api_bind_options, options => $heat_api_options, + bind_options => $heat_api_bind_options, firewall_settings => $firewall_settings, } if 'ssl' in $heat_cfn_bind_options { - $heat_cfn_options = { - 'reqadd' => 'X-Forwarded-Proto:\ https if { ssl_fc }' } + $heat_cfn_options = merge($common_tcp_options, { + 'reqadd' => 'X-Forwarded-Proto:\ https if { ssl_fc }', + }) } else { - $heat_cfn_options = { } + $heat_cfn_options = $common_tcp_options } cloud::loadbalancer::binding { 'heat_cfn_api_cluster': ip => $heat_cfn_api, @@ -866,10 +875,11 @@ class cloud::loadbalancer( firewall_settings => $firewall_settings, } if 'ssl' in $heat_cloudwatch_bind_options { - $heat_cloudwatch_options = { - 'reqadd' => 'X-Forwarded-Proto:\ https if { ssl_fc }' } + $heat_cloudwatch_options = merge($common_tcp_options, { + 'reqadd' => 'X-Forwarded-Proto:\ https if { ssl_fc }', + }) } else { - $heat_cloudwatch_options = { } + $heat_cloudwatch_options = $common_tcp_options } cloud::loadbalancer::binding { 'heat_cloudwatch_api_cluster': ip => $heat_cloudwatch_api, @@ -953,8 +963,8 @@ class cloud::loadbalancer( 'mode' => 'tcp', 'balance' => 'roundrobin', 'option' => ['tcpka', 'tcplog', 'httpchk'], #httpchk mandatory expect 200 on port 9000 - 'timeout client' => $galera_timeout, - 'timeout server' => $galera_timeout, + 'timeout client' => '90m', + 'timeout server' => '90m', }, bind_options => $galera_bind_options, } @@ -976,8 +986,8 @@ class cloud::loadbalancer( 'mode' => 'tcp', 'balance' => 'roundrobin', 'option' => ['tcpka', 'tcplog', 'httpchk'], #httpchk mandatory expect 200 on port 9000 - 'timeout client' => $galera_timeout, - 'timeout server' => $galera_timeout, + 'timeout client' => '90m', + 'timeout server' => '90m', }, bind_options => $galera_bind_options, } diff --git a/spec/classes/cloud_loadbalancer_spec.rb b/spec/classes/cloud_loadbalancer_spec.rb index 5cb77d4f..2ccbc287 100644 --- a/spec/classes/cloud_loadbalancer_spec.rb +++ b/spec/classes/cloud_loadbalancer_spec.rb @@ -269,8 +269,8 @@ describe 'cloud::loadbalancer' do 'mode' => 'tcp', 'balance' => 'source', 'option' => ['tcpka', 'tcplog', 'forwardfor'], - 'timeout server' => '120m', - 'timeout client' => '120m' + 'timeout server' => '90m', + 'timeout client' => '90m' } )} end @@ -289,8 +289,8 @@ describe 'cloud::loadbalancer' do 'mode' => 'tcp', 'balance' => 'source', 'option' => ['tcpka', 'tcplog', 'forwardfor'], - 'timeout server' => '120m', - 'timeout client' => '120m' + 'timeout server' => '90m', + 'timeout client' => '90m' } )} end @@ -404,10 +404,11 @@ describe 'cloud::loadbalancer' do :ipaddress => [params[:vip_public_ip]], :ports => '8774', :options => { - 'mode' => 'http', - 'option' => ['tcpka','forwardfor','tcplog','httpchk'], - 'http-check' => 'expect ! rstatus ^5', - 'balance' => 'roundrobin', + 'mode' => 'tcp', + 'balance' => 'source', + 'option' => ['tcpka', 'tcplog', 'forwardfor'], + 'timeout server' => '90m', + 'timeout client' => '90m' }, :bind_options => ['ssl', 'crt'] )} @@ -423,10 +424,11 @@ describe 'cloud::loadbalancer' do :ipaddress => [params[:vip_public_ip]], :ports => '8776', :options => { - 'mode' => 'http', - 'option' => ['tcpka','forwardfor','tcplog', 'httpchk'], - 'http-check' => 'expect ! rstatus ^5', - 'balance' => 'roundrobin', + 'mode' => 'tcp', + 'balance' => 'source', + 'option' => ['tcpka', 'tcplog', 'forwardfor'], + 'timeout server' => '90m', + 'timeout client' => '90m' }, :bind_options => ['something not secure'] )} @@ -499,11 +501,12 @@ describe 'cloud::loadbalancer' do :ipaddress => [params[:vip_public_ip]], :ports => '8004', :options => { + 'mode' => 'tcp', 'reqadd' => 'X-Forwarded-Proto:\ https if { ssl_fc }', - 'mode' => 'http', - 'option' => ['tcpka','forwardfor','tcplog', 'httpchk'], - 'http-check' => 'expect ! rstatus ^5', - 'balance' => 'roundrobin' + 'balance' => 'source', + 'option' => ['tcpka', 'tcplog', 'forwardfor'], + 'timeout server' => '90m', + 'timeout client' => '90m' }, :bind_options => ['ssl', 'crt'] )} @@ -519,6 +522,8 @@ describe 'cloud::loadbalancer' do 'mode' => 'tcp', 'balance' => 'roundrobin', 'option' => ['tcpka', 'tcplog', 'forwardfor'], + 'timeout server' => '90m', + 'timeout client' => '90m', } )} end