cookbook-ceph/libraries/default.rb

require 'ipaddr'

def is_crowbar?()
  return defined?(Chef::Recipe::Barclamp) != nil
end

def get_mon_nodes(extra_search=nil)
  if is_crowbar?
    mon_roles = search(:role, 'name:crowbar-* AND run_list:role\[ceph-mon\]')
    if not mon_roles.empty?
      search_string = mon_roles.map { |role_object| "role:"+role_object.name }.join(' OR ')
      search_string = "(#{search_string}) AND ceph_config_environment:#{node['ceph']['config']['environment']}"
    end
  else
    search_string = "role:ceph-mon AND chef_environment:#{node.chef_environment}"
  end

  if not extra_search.nil?
    search_string = "(#{search_string}) AND (#{extra_search})"
  end
  mons = search(:node, search_string)
  return mons
end

# If public_network is specified
# we need to search for the monitor IP
# in the node environment.
# 1. We look if the network is IPv6 or IPv4
# 2. We look for a route matching the network
# 3. We grab the IP and return it with the port
def find_node_ip_in_network(network, nodeish=nil)
  nodeish = node unless nodeish
  net = IPAddr.new(network)
  node["network"]["interfaces"].each do |iface|
    if iface[1]["routes"].nil?
      next
    end
    if net.ipv4?
      iface[1]["routes"].each_with_index do |route, index|
        if iface[1]["routes"][index]["destination"] == network
          return "#{iface[1]["routes"][index]["src"]}:6789"
        end
      end
    else
      # Here we are getting an IPv6. We assume that
      # the configuration is stateful.
      # For this configuration to not fail in a stateless
      # configuration, you should run:
      #  echo "0" > /proc/sys/net/ipv6/conf/*/use_tempaddr
      # on each server, this will disabe temporary addresses
      # See: http://en.wikipedia.org/wiki/IPv6_address#Temporary_addresses
      iface[1]["routes"].each_with_index do |route, index|
        if iface[1]["routes"][index]["destination"] == network
          iface[1]["addresses"].each do |k,v|
            if v["scope"] == "Global" and v["family"] == "inet6"
              return "[#{k}]:6789"
            end
          end
        end
      end
    end
  end
end

def get_mon_addresses()
  mon_ips = []

  if File.exists?("/var/run/ceph/ceph-mon.#{node['hostname']}.asok")
    mon_ips = get_quorum_members_ips()
  else
    mons = []
    # make sure if this node runs ceph-mon, it's always included even if
    # search is laggy; put it first in the hopes that clients will talk
    # primarily to local node
    if node['roles'].include? 'ceph-mon'
      mons << node
    end

    mons += get_mon_nodes()
    if is_crowbar?
      mon_ips = mons.map { |node| Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "admin").address }
    else
      if node['ceph']['config']['global'] && node['ceph']['config']['global']['public network']
        mon_ips = mons.map { |nodeish| find_node_ip_in_network(node['ceph']['config']['global']['public network'], nodeish) }
      else
        mon_ips = mons.map { |node| node['ipaddress'] + ":6789" }
      end
    end
  end
  return mon_ips.uniq
end

def get_quorum_members_ips()
  mon_ips = []
  mon_status = %x[ceph --admin-daemon /var/run/ceph/ceph-mon.#{node['hostname']}.asok mon_status]
  raise 'getting quorum members failed' unless $?.exitstatus == 0

  mons = JSON.parse(mon_status)['monmap']['mons']
  mons.each do |k|
    mon_ips.push(k['addr'][0..-3])
  end
  return mon_ips
end

QUORUM_STATES = ['leader', 'peon']
def have_quorum?()
    # "ceph auth get-or-create-key" would hang if the monitor wasn't
    # in quorum yet, which is highly likely on the first run. This
    # helper lets us delay the key generation into the next
    # chef-client run, instead of hanging.
    #
    # Also, as the UNIX domain socket connection has no timeout logic
    # in the ceph tool, this exits immediately if the ceph-mon is not
    # running for any reason; trying to connect via TCP/IP would wait
    # for a relatively long timeout.
    mon_status = %x[ceph --admin-daemon /var/run/ceph/ceph-mon.#{node['hostname']}.asok mon_status]
    raise 'getting monitor state failed' unless $?.exitstatus == 0
    state = JSON.parse(mon_status)['state']
    return QUORUM_STATES.include?(state)
end

def service_type()
  case node['platform']
  when 'ubuntu'
    return "upstart"
  else
    return "sysvinit"
  end
end