From 24bd6bc3d81394fc623e68039eb4ed23ecb9f775 Mon Sep 17 00:00:00 2001 From: Mehdi Abaakouk Date: Thu, 18 Sep 2014 09:24:12 +0200 Subject: [PATCH 1/3] [galera] Improve clustercheck script --- templates/database/clustercheck.erb | 59 +++++++++++++++++++---------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/templates/database/clustercheck.erb b/templates/database/clustercheck.erb index df8bd288..5336bac6 100644 --- a/templates/database/clustercheck.erb +++ b/templates/database/clustercheck.erb @@ -4,35 +4,54 @@ # # Script to make a proxy (ie HAProxy) capable of monitoring Percona XtraDB Cluster nodes properly # -# Author: Olaf van Zandwijk olaf.vanzandwijk@nedap.com +# Author: Olaf van Zandwijk +# Mehdi Abaakouk +# # Documentation and download: https://github.com/olafz/percona-clustercheck # # Based on the original script from Unai Rodriguez # +MYSQL_USERNAME='<%= @galera_clustercheck_dbuser %>' +MYSQL_PASSWORD='<%= @galera_clustercheck_dbpassword %>' -MYSQL_USERNAME="<%= @galera_clustercheck_dbuser %>" -MYSQL_PASSWORD="<%= @galera_clustercheck_dbpassword %>" +TIMEOUT=10 ERR_FILE="/dev/null" AVAILABLE_WHEN_DONOR=0 -# -# Perform the query to check the wsrep_local_state -# -WSREP_STATUS=`mysql --user=${MYSQL_USERNAME} --password=${MYSQL_PASSWORD} -e "SHOW STATUS LIKE 'wsrep_local_state';" 2>${ERR_FILE} | awk '{if (NR!=1){print $2}}' 2>${ERR_FILE}` +MYSQL_CMDLINE="mysql -nNE --connect-timeout=$TIMEOUT --user=${MYSQL_USERNAME} --password=${MYSQL_PASSWORD} " -if [[ "${WSREP_STATUS}" == "4" ]] || [[ "${WSREP_STATUS}" == "2" && ${AVAILABLE_WHEN_DONOR} == 1 ]] -then - # Percona XtraDB Cluster node local state is 'Synced' == return HTTP 200 - /bin/echo -en "HTTP/1.1 200 OK\r\n" +mysql_get_status(){ + ( $MYSQL_CMDLINE -e "SHOW STATUS LIKE '$1';" | tail -1 ) 2>>${ERR_FILE} +} +mysql_get_var(){ + ( $MYSQL_CMDLINE -e "SHOW GLOBAL VARIABLES LIKE '$1';" | tail -1 ) 2>>${ERR_FILE} +} + +http_response(){ + status=$1 + shift + msg="$@" + if [ "$status" == 200 ]; then + /bin/echo -en "HTTP/1.1 200 OK\r\n" + else + /bin/echo -en "HTTP/1.1 503 Service Unavailable\r\n" + fi /bin/echo -en "Content-Type: text/plain\r\n" /bin/echo -en "\r\n" - /bin/echo -en "Mariadb Cluster Node is synced.\r\n" + /bin/echo -en "$msg\r\n" /bin/echo -en "\r\n" -else - # Percona XtraDB Cluster node local state is not 'Synced' == return HTTP 503 - /bin/echo -en "HTTP/1.1 503 Service Unavailable\r\n" - /bin/echo -en "Content-Type: text/plain\r\n" - /bin/echo -en "\r\n" - /bin/echo -en "Mariadb Cluster Node is not synced.\r\n" - /bin/echo -en "\r\n" -fi +} + + +WSREP_LOCAL_STATE=$(mysql_get_status wsrep_local_state) +WSREP_READY=$(mysql_get_status wsrep_ready) +WSREP_CONNECTED=$(mysql_get_status wsrep_connected) +READY_ONLY=$(mysql_get_var read_only) + +case ${AVAILABLE_WHEN_DONOR}-${WSREP_LOCAL_STATE}-${WSREP_READY}-${WSREP_CONNECTED}-${READY_ONLY} in + 1-2-ON-ON-OFF|0-4-ON-ON-OFF) http_response 200 "Mariadb Cluster Node is synced, ready and connected." ;; + *-*-OFF-*-*) http_response 503 "Mariadb Cluster Node is not ready." ;; + *-*-*-OFF-*) http_response 503 "Mariadb Cluster Node is not connected" ;; + *-*-*-*-ON) http_response 503 "Mariadb Cluster Node is readonly" ;; + *) http_response 503 "Mariadb Cluster Node is not synced" ;; +esac From 253aaeb13888993b72438032aaaaca139c6e4113 Mon Sep 17 00:00:00 2001 From: Mehdi Abaakouk Date: Thu, 18 Sep 2014 10:42:35 +0200 Subject: [PATCH 2/3] [galera] Some master/slave tweaks This adds some tweaks for galera master/slave setup It also allows to customize the galera gcache size, This value depends mainly of the database traffic. But the default value of 128M is clearly to low for us, this means if a down node comeback and the data that need to be transfered is greater than 128MB a full sync if done instead of an incremental. So we use 1G by default and that should changed in case of a deployement with a big database workload. --- manifests/database/sql.pp | 1 + spec/classes/cloud_database_sql_spec.rb | 5 +++-- templates/database/mysql.conf.erb | 10 ++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/manifests/database/sql.pp b/manifests/database/sql.pp index 34706a61..194f120e 100644 --- a/manifests/database/sql.pp +++ b/manifests/database/sql.pp @@ -26,6 +26,7 @@ class cloud::database::sql ( $service_provider = 'sysv', $galera_master_name = 'mgmt001', $galera_internal_ips = ['127.0.0.1'], + $galera_gcache = '1G', $keystone_db_host = '127.0.0.1', $keystone_db_user = 'keystone', $keystone_db_password = 'keystonepassword', diff --git a/spec/classes/cloud_database_sql_spec.rb b/spec/classes/cloud_database_sql_spec.rb index 01386658..6b76f2a9 100644 --- a/spec/classes/cloud_database_sql_spec.rb +++ b/spec/classes/cloud_database_sql_spec.rb @@ -32,6 +32,7 @@ describe 'cloud::database::sql' do :api_eth => '10.0.0.1', :galera_master_name => 'os-ci-test1', :galera_internal_ips => ['10.0.0.1','10.0.0.2','10.0.0.3'], + :galera_gcache => '1G', :keystone_db_host => '10.0.0.1', :keystone_db_user => 'keystone', :keystone_db_password => 'secrete', @@ -88,8 +89,8 @@ describe 'cloud::database::sql' do context 'configure mysqlchk http replication' do it { should contain_file('/etc/xinetd.d/mysqlchk').with_mode('0755') } it { should contain_file('/usr/bin/clustercheck').with_mode('0755') } - it { should contain_file('/usr/bin/clustercheck').with_content(/MYSQL_USERNAME="#{params[:galera_clustercheck_dbuser]}"/)} - it { should contain_file('/usr/bin/clustercheck').with_content(/MYSQL_PASSWORD="#{params[:galera_clustercheck_dbpassword]}"/)} + it { should contain_file('/usr/bin/clustercheck').with_content(/MYSQL_USERNAME='#{params[:galera_clustercheck_dbuser]}'/)} + it { should contain_file('/usr/bin/clustercheck').with_content(/MYSQL_PASSWORD='#{params[:galera_clustercheck_dbpassword]}'/)} it { should contain_file('/etc/xinetd.d/mysqlchk').with_content(/bind = #{params[:galera_clustercheck_ipaddress]}/)} end # configure mysqlchk http replication diff --git a/templates/database/mysql.conf.erb b/templates/database/mysql.conf.erb index fca71da0..46ea36a6 100644 --- a/templates/database/mysql.conf.erb +++ b/templates/database/mysql.conf.erb @@ -49,14 +49,16 @@ wsrep_provider = "<%= @wsrep_provider %>" wsrep_cluster_name = "galera_cluster" wsrep_cluster_address = "gcomm://<%= @gcomm_definition %>" wsrep_sst_auth = root:<%= @mysql_root_password %> -wsrep_certify_nonPK = 1 -wsrep_convert_LOCK_to_trx = 0 -wsrep_auto_increment_control = 1 wsrep_drupal_282555_workaround = 0 -wsrep_causal_reads = 0 wsrep_sst_method = rsync wsrep_node_address = "<%= @api_eth %>" wsrep_node_incoming_address = "<%= @api_eth %>" +# This is the minimal value (proc*2) +wsrep_slave_threads = "<%= @processorcount.to_i * 2 %>" + +# Thoses TWEAK assume that the galera cluster is used in master/slave mode +wsrep_provider_options = "gcache.size=<%= @galera_gcache %>;gcs.fc_master_slave=1;gcs.fc_limit=256;gcs.fc_factor=0.9" + # this value here are used by /usr/bin/innobackupex # and wsrep_sst_xtrabackup take only one configuration file and use the last one # (/etc/mysql/my.cnf is not used) From 8ced994f79b737093bc82e384eee7e9032e1d455 Mon Sep 17 00:00:00 2001 From: Mehdi Abaakouk Date: Thu, 18 Sep 2014 11:05:33 +0200 Subject: [PATCH 3/3] [galera] use same value for galera and haproxy max conn Use same value for galera and haproxy max connection The extra connection will be dropped by haproxy instead of galera. --- manifests/loadbalancer.pp | 1 + spec/classes/cloud_loadbalancer_spec.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/manifests/loadbalancer.pp b/manifests/loadbalancer.pp index 7c74e58e..f81c2be3 100644 --- a/manifests/loadbalancer.pp +++ b/manifests/loadbalancer.pp @@ -491,6 +491,7 @@ class cloud::loadbalancer( ipaddress => $galera_ip, ports => 3306, options => { + 'maxconn' => '1000', 'mode' => 'tcp', 'balance' => 'roundrobin', 'option' => ['tcpka', 'tcplog', 'httpchk'], #httpchk mandatory expect 200 on port 9000 diff --git a/spec/classes/cloud_loadbalancer_spec.rb b/spec/classes/cloud_loadbalancer_spec.rb index dc5f3ad0..a866e366 100644 --- a/spec/classes/cloud_loadbalancer_spec.rb +++ b/spec/classes/cloud_loadbalancer_spec.rb @@ -206,6 +206,7 @@ describe 'cloud::loadbalancer' do :ipaddress => params[:galera_ip], :ports => '3306', :options => { + 'maxconn' => '1000', 'mode' => 'tcp', 'balance' => 'roundrobin', 'option' => ['tcpka','tcplog','httpchk'],