337 lines
11 KiB
Ruby
337 lines
11 KiB
Ruby
require 'ipaddr'
|
|
require 'json'
|
|
require 'chef/mixin/shell_out'
|
|
include Chef::Mixin::ShellOut
|
|
|
|
def crowbar?
|
|
!defined?(Chef::Recipe::Barclamp).nil?
|
|
end
|
|
|
|
def mon_nodes
|
|
if crowbar?
|
|
mon_roles = search(:role, 'name:crowbar-* AND run_list:role\[ceph*-mon\]')
|
|
unless mon_roles.empty?
|
|
search_string = mon_roles.map { |role_object| 'roles:' + role_object.name }.join(' OR ')
|
|
search_string = "(#{search_string}) AND ceph_config_environment:#{node['ceph']['config']['environment']}"
|
|
end
|
|
else
|
|
#search_string = "ceph_is_mon:true AND chef_environment:#{node.chef_environment}"
|
|
search_string = "ceph_is_mon:true AND ceph_config_fsid:#{node["ceph"]["config"]["fsid"]}"
|
|
end
|
|
|
|
if use_cephx? && !node['ceph']['encrypted_data_bags']
|
|
search_string = "(#{search_string}) AND (ceph_bootstrap_osd_key:*)"
|
|
end
|
|
search(:node, search_string)
|
|
end
|
|
|
|
def osd_secret
|
|
if node['ceph']['encrypted_data_bags']
|
|
secret = Chef::EncryptedDataBagItem.load_secret(node['ceph']['osd']['secret_file'])
|
|
return Chef::EncryptedDataBagItem.load('ceph', 'osd', secret)['secret']
|
|
else
|
|
return mon_nodes[0]['ceph']['bootstrap_osd_key']
|
|
end
|
|
end
|
|
|
|
# If public_network is specified
|
|
# we need to search for the monitor IP
|
|
# in the node environment.
|
|
# 1. We look if the network is IPv6 or IPv4
|
|
# 2. We look for a route matching the network
|
|
# 3. We grab the IP and return it with the port
|
|
def find_node_ip_in_network(network, nodeish = nil)
|
|
nodeish = node unless nodeish
|
|
net = IPAddr.new(network)
|
|
nodeish['network']['interfaces'].each do |_iface, addrs|
|
|
addresses = addrs['addresses'] || []
|
|
addresses.each do |ip, params|
|
|
return ip_address_to_ceph_address(ip, params) if ip_address_in_network?(ip, params, net)
|
|
end
|
|
end
|
|
nil
|
|
end
|
|
|
|
def ip_address_in_network?(ip, params, net)
|
|
if params['family'] == 'inet'
|
|
net.include?(ip) && params.key?('broadcast') # is primary ip on iface
|
|
elsif params['family'] == 'inet6'
|
|
net.include?(ip)
|
|
else
|
|
false
|
|
end
|
|
end
|
|
|
|
def ip_address_to_ceph_address(ip, params)
|
|
if params['family'].eql?('inet6')
|
|
return "[#{ip}]:6789"
|
|
elsif params['family'].eql?('inet')
|
|
return "#{ip}:6789"
|
|
end
|
|
end
|
|
|
|
def mon_addresses
|
|
mon_ips = []
|
|
|
|
if File.exist?("/var/run/ceph/ceph-mon.#{node['hostname']}.asok")
|
|
mon_ips = quorum_members_ips
|
|
else
|
|
mons = []
|
|
# make sure if this node runs ceph-mon, it's always included even if
|
|
# search is laggy; put it first in the hopes that clients will talk
|
|
# primarily to local node
|
|
mons << node if node['ceph']['is_mon']
|
|
|
|
mons += mon_nodes
|
|
if crowbar?
|
|
mon_ips = mons.map { |node| Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, 'admin').address }
|
|
else
|
|
if node['ceph']['config']['global'] && node['ceph']['config']['global']['public network']
|
|
mon_ips = mons.map { |nodeish| find_node_ip_in_network(node['ceph']['config']['global']['public network'], nodeish) }
|
|
else
|
|
mon_ips = mons.map { |node| node['ipaddress'] + ':6789' }
|
|
end
|
|
end
|
|
end
|
|
mon_ips.reject { |m| m.nil? }.uniq
|
|
end
|
|
|
|
def mon_secret
|
|
if node['ceph']['encrypted_data_bags']
|
|
secret = Chef::EncryptedDataBagItem.load_secret(node['ceph']['mon']['secret_file'])
|
|
Chef::EncryptedDataBagItem.load('ceph', 'mon', secret)['secret']
|
|
elsif !mon_nodes.empty?
|
|
mon_nodes[0]['ceph']['monitor-secret']
|
|
elsif node['ceph']['monitor-secret']
|
|
node['ceph']['monitor-secret']
|
|
elsif mon_master['hostname'] != node['hostname']
|
|
mon_nodes[0]['ceph']['monitor-secret']
|
|
else
|
|
Chef::Log.info('No monitor secret found')
|
|
nil
|
|
end
|
|
end
|
|
|
|
def pg_creating?
|
|
pg_creating_flag = ''
|
|
if pg_creating_flag.empty?
|
|
pg_creating_flag = Mixlib::ShellOut.new('ceph -s | grep "creating"').run_command.stdout.strip
|
|
end
|
|
pg_creating_flag.include?('creating')
|
|
end
|
|
|
|
def mon_master
|
|
search_string2 = "run_list:role\\[ceph*-mon\\] AND chef_environment:#{node.chef_environment} AND tags:mon_master"
|
|
mon_master_node = search(:node, search_string2)
|
|
if mon_master_node.empty?
|
|
search_string2 = "run_list:role\\[ceph*-mon\\] AND chef_environment:#{node.chef_environment}"
|
|
all_mons = search(:node, search_string2)
|
|
|
|
# TODO(weidong): should add empty? checking later.
|
|
mons_sort = all_mons.sort_by { |a| a.name }
|
|
if mons_sort[0].name == node.name
|
|
node.tags << 'mon_master' unless node.tags.include?("mon_master")
|
|
node.save
|
|
end
|
|
mons_sort[0]
|
|
else
|
|
mon_master_node[0]
|
|
end
|
|
end
|
|
|
|
def mon_init_member
|
|
search_string3 = "run_list:role\\[ceph*-mon\\] AND chef_environment:#{node.chef_environment}"
|
|
all_mons = search(:node, search_string3)
|
|
mons_sort = all_mons.sort_by { |a| a.name }
|
|
end
|
|
|
|
def mon_init_member_name
|
|
mon_list = mon_init_member
|
|
mon_list.map { |a| a.name }.sort unless mon_list.nil?
|
|
end
|
|
|
|
#search SSD device
|
|
def ssd_device
|
|
ssd_device = []
|
|
node['block_device'].each do |device|
|
|
device_name = device[0]
|
|
if device_name.include?"sd"
|
|
device_ssd_flag = Mixlib::ShellOut.new("cat /sys/block/#{device_name}/queue/rotational").run_command.stdout.strip
|
|
if device_ssd_flag == "0"
|
|
ssd_device << "/dev/#{device_name}"
|
|
end
|
|
else
|
|
next
|
|
end
|
|
end
|
|
ssd_device
|
|
end
|
|
|
|
def quorum_members_ips
|
|
mon_ips = []
|
|
cmd = Mixlib::ShellOut.new("ceph --admin-daemon /var/run/ceph/ceph-mon.#{node['hostname']}.asok mon_status")
|
|
cmd.run_command
|
|
cmd.error!
|
|
|
|
mons = JSON.parse(cmd.stdout)['monmap']['mons']
|
|
mons.each do |k|
|
|
mon_ips.push(k['addr'][0..-3])
|
|
end
|
|
mon_ips
|
|
end
|
|
|
|
QUORUM_STATES = %w(leader, peon)
|
|
def quorum?
|
|
# "ceph auth get-or-create-key" would hang if the monitor wasn't
|
|
# in quorum yet, which is highly likely on the first run. This
|
|
# helper lets us delay the key generation into the next
|
|
# chef-client run, instead of hanging.
|
|
#
|
|
# Also, as the UNIX domain socket connection has no timeout logic
|
|
# in the ceph tool, this exits immediately if the ceph-mon is not
|
|
# running for any reason; trying to connect via TCP/IP would wait
|
|
# for a relatively long timeout.
|
|
|
|
cmd = Mixlib::ShellOut.new("ceph --admin-daemon /var/run/ceph/ceph-mon.#{node['hostname']}.asok mon_status")
|
|
cmd.run_command
|
|
cmd.error!
|
|
|
|
state = JSON.parse(cmd.stdout)['state']
|
|
QUORUM_STATES.include?(state)
|
|
end
|
|
|
|
# Cephx is on by default, but users can disable it.
|
|
# type can be one of 3 values: cluster, service, or client. If the value is none of the above, set it to cluster
|
|
def use_cephx?(type = nil)
|
|
# Verify type is valid
|
|
type = 'cluster' if %w(cluster service client).index(type).nil?
|
|
|
|
# CephX is enabled if it's not configured at all, or explicity enabled
|
|
node['ceph']['config'].nil? ||
|
|
node['ceph']['config']['global'].nil? ||
|
|
node['ceph']['config']['global']["auth #{type} required"] == 'cephx'
|
|
end
|
|
|
|
#current partion number of a given device
|
|
def partition_num(device)
|
|
cmd = "parted #{device} --script -- p | awk '{print $1}'"
|
|
rc = shell_out(cmd)
|
|
p_num = rc.stdout.split.select{|e| e[/\d/]}
|
|
if p_num.include? "Number"
|
|
last_num = 0
|
|
Chef::Log.info("There is not any partition created at #{resource.device} yet.")
|
|
end
|
|
p_num
|
|
end
|
|
|
|
#partion start size of a given device
|
|
def partition_start_size(device)
|
|
cmd = "parted #{device} --script -- p | awk '{print $3}' | tail -n 2"
|
|
rc = shell_out(cmd)
|
|
device_start_size = rc.stdout.split[0]
|
|
if device_start_size.include? "End"
|
|
device_start_size = 0
|
|
end
|
|
if device_start_size == 0
|
|
device_start_size
|
|
elsif device_start_size.include?('KB')
|
|
device_start_size = eval(device_start_size.gsub(/[A-Z]/,''))/1000
|
|
elsif device_start_size.include?('GB')
|
|
device_start_size = eval(device_start_size.gsub(/[A-Z]/,''))*1000
|
|
elsif device_start_size.include?('MB')
|
|
device_start_size = eval(device_start_size.gsub(/[A-Z]/,''))
|
|
elsif device_start_size.include?('TB')
|
|
device_start_size = eval(device_start_size.gsub(/[A-Z]/,''))*1000000
|
|
end
|
|
device_start_size
|
|
end
|
|
|
|
def disk_total_size(device)
|
|
cmd = "parted #{device} --script -- p | grep #{device} | cut -f 2 -d ':'"
|
|
rc = shell_out(cmd)
|
|
device_total_size = rc.stdout.split[0]
|
|
if device_total_size.include?('GB')
|
|
device_total_size = eval(device_total_size.gsub(/[A-Z]/,''))*1000
|
|
elsif device_total_size.include?('MB')
|
|
device_total_size = eval(device_total_size.gsub(/[A-Z]/,''))
|
|
elsif device_total_size.include?('TB')
|
|
device_total_size = eval(device_total_size.gsub(/[A-Z]/,''))*1000000
|
|
end
|
|
device_total_size
|
|
end
|
|
|
|
def mklabel(device)
|
|
queryresult = %x{parted #{device} --script -- print |grep 'Partition Table: gpt'}
|
|
if not queryresult.include?('gpt')
|
|
cmd = "parted #{device} --script -- mklabel gpt"
|
|
rc = shell_out(cmd)
|
|
if not rc.exitstatus.eql?(0)
|
|
Chef::Log.error("Creating disk label was failed.")
|
|
end
|
|
end
|
|
end
|
|
|
|
def mkpart(device)
|
|
device_total_size = disk_total_size(device)
|
|
device_start_size = partition_start_size(device) + 100
|
|
if node['ceph']['config']['osd']['osd journal size']
|
|
osd_journal_size = node['ceph']['config']['osd']['osd journal size'].to_i
|
|
elsif node['ceph']['config']['global']['osd journal size']
|
|
osd_journal_size = node['ceph']['config']['global']['osd journal size'].to_i
|
|
else
|
|
osd_journal_size = 5120
|
|
end
|
|
device_end_size = device_start_size + osd_journal_size
|
|
if device_start_size < device_total_size
|
|
p_num_old = partition_num(device)
|
|
if device_total_size > device_end_size
|
|
output = %x{parted #{device} --script -- mkpart osd_journal #{device_start_size.to_s} #{device_end_size.to_s}}
|
|
else
|
|
output = %x{parted #{device} --script -- mkpart osd_journal #{device_start_size.to_s} 100%}
|
|
end
|
|
output = %x{partx -a #{device} > /dev/null 2>&1}
|
|
p_num_new = partition_num(device)
|
|
p_num = (p_num_new - p_num_old)[0]
|
|
if p_num.nil?
|
|
Chef::Log.error("Making partition was failed.")
|
|
else
|
|
device_return = device+p_num
|
|
%x{mkfs.xfs #{device_return}}
|
|
device_return
|
|
end
|
|
end
|
|
end
|
|
|
|
def create_disk_partion(device)
|
|
mklabel(device)
|
|
mkpart(device)
|
|
end
|
|
|
|
def selinux_disabled?
|
|
selinux_status = Mixlib::ShellOut.new('sestatus').run_command.stdout.strip
|
|
selinux_status.include?('disabled')
|
|
end
|
|
|
|
def node_election(role, tag, chef_environment = nil)
|
|
chef_environment = chef_environment || node.chef_environment
|
|
master = search(:node, "run_list:role\\[#{role}\\] AND \
|
|
chef_environment:#{chef_environment} AND \
|
|
tags:#{tag}") || []
|
|
if master.empty?
|
|
nodes = search(:node, "run_list:role\\[#{role}\\] AND \
|
|
chef_environment:#{chef_environment}") || []
|
|
if !nodes.empty?
|
|
nodes = nodes.sort_by { |node| node.name } unless nodes.empty?
|
|
if node['hostname'].eql?(nodes[0]['hostname'])
|
|
node.tags << tag unless node.tags.include?(tag)
|
|
node.save
|
|
end
|
|
return nodes[0]
|
|
else
|
|
node
|
|
end
|
|
else
|
|
return master[0]
|
|
end
|
|
end
|