e1d544adc2
Currently, the list of minions is statically specified in master node, and the list of IP addresses of minion nodes is passed to master node through user_data. This approach doesn't work well on scaling. For example, adding a minion node through stack-update will cause an update on user_data, which cause the master node to be replaced. This commit addresses this issue by dynamically register minion node. There are two cases: * Scale out: the new minion nodes will register themselves to the cluster when they are booting. * Scale in: a subset of minion nodes are deleted by Heat, and the master node will discover it and automatically deregister the deleted nodes.
340 lines
9.0 KiB
YAML
340 lines
9.0 KiB
YAML
heat_template_version: 2013-05-23
|
|
|
|
description: >
|
|
This template will boot a Kubernetes cluster with one or more
|
|
minions (as specified by the number_of_minions parameter, which
|
|
defaults to "2").
|
|
|
|
parameters:
|
|
|
|
#
|
|
# REQUIRED PARAMETERS
|
|
#
|
|
ssh_key_name:
|
|
type: string
|
|
description: name of ssh key to be provisioned on our server
|
|
|
|
external_network_id:
|
|
type: string
|
|
description: uuid of a network to use for floating ip addresses
|
|
|
|
#
|
|
# OPTIONAL PARAMETERS
|
|
#
|
|
server_image:
|
|
type: string
|
|
default: centos-atomic
|
|
description: glance image used to boot the server
|
|
|
|
master_flavor:
|
|
type: string
|
|
default: m1.small
|
|
description: flavor to use when booting the server
|
|
|
|
server_flavor:
|
|
type: string
|
|
default: m1.small
|
|
description: flavor to use when booting the server
|
|
|
|
dns_nameserver:
|
|
type: string
|
|
description: address of a dns nameserver reachable in your environment
|
|
default: 8.8.8.8
|
|
|
|
number_of_minions:
|
|
type: string
|
|
description: how many kubernetes minions to spawn
|
|
default: 1
|
|
|
|
fixed_network_cidr:
|
|
type: string
|
|
description: network range for fixed ip network
|
|
default: "10.0.0.0/24"
|
|
|
|
portal_network_cidr:
|
|
type: string
|
|
description: >
|
|
address range used by kubernetes for service portals
|
|
default: "10.254.0.0/16"
|
|
|
|
flannel_network_cidr:
|
|
type: string
|
|
description: network range for flannel overlay network
|
|
default: "10.100.0.0/16"
|
|
|
|
flannel_network_subnetlen:
|
|
type: string
|
|
description: size of subnet assigned to each minion
|
|
default: 24
|
|
|
|
flannel_use_vxlan:
|
|
type: string
|
|
description: >
|
|
if true use the vxlan backend, otherwise use the default
|
|
udp backend
|
|
default: "false"
|
|
constraints:
|
|
- allowed_values: ["true", "false"]
|
|
|
|
kube_allow_priv:
|
|
type: string
|
|
description: >
|
|
whether or not kubernetes should permit privileged containers.
|
|
default: "true"
|
|
constraints:
|
|
- allowed_values: ["true", "false"]
|
|
|
|
docker_volume_size:
|
|
type: string
|
|
description: >
|
|
size of a cinder volume to allocate to docker for container/image
|
|
storage
|
|
default: "25"
|
|
|
|
resources:
|
|
|
|
master_wait_handle:
|
|
type: "AWS::CloudFormation::WaitConditionHandle"
|
|
|
|
master_wait_condition:
|
|
type: "AWS::CloudFormation::WaitCondition"
|
|
depends_on:
|
|
- kube_master
|
|
properties:
|
|
Handle:
|
|
get_resource: master_wait_handle
|
|
Timeout: "6000"
|
|
|
|
######################################################################
|
|
#
|
|
# network resources. allocate a network and router for our server.
|
|
# it would also be possible to take advantage of existing network
|
|
# resources (and have the deployer provide network and subnet ids,
|
|
# etc, as parameters), but I wanted to minmize the amount of
|
|
# configuration necessary to make this go.
|
|
|
|
fixed_network:
|
|
type: "OS::Neutron::Net"
|
|
|
|
# This is the subnet on which we will deploy our server.
|
|
fixed_subnet:
|
|
type: "OS::Neutron::Subnet"
|
|
properties:
|
|
cidr: {get_param: fixed_network_cidr}
|
|
network_id:
|
|
get_resource: fixed_network
|
|
dns_nameservers:
|
|
- get_param: dns_nameserver
|
|
|
|
# create a router attached to the external network provided as a
|
|
# parameter to this stack.
|
|
extrouter:
|
|
type: "OS::Neutron::Router"
|
|
properties:
|
|
external_gateway_info:
|
|
network:
|
|
get_param: external_network_id
|
|
|
|
# attached fixed_subnet to our extrouter router.
|
|
extrouter_inside:
|
|
type: "OS::Neutron::RouterInterface"
|
|
properties:
|
|
router_id:
|
|
get_resource: extrouter
|
|
subnet_id:
|
|
get_resource:
|
|
fixed_subnet
|
|
|
|
######################################################################
|
|
#
|
|
# security groups. we need to permit network traffic of various
|
|
# sorts.
|
|
#
|
|
|
|
# permit ssh access
|
|
secgroup_base:
|
|
type: "OS::Neutron::SecurityGroup"
|
|
properties:
|
|
rules:
|
|
- protocol: icmp
|
|
- protocol: tcp
|
|
port_range_min: 22
|
|
port_range_max: 22
|
|
|
|
# open ports for kubernetes and etcd
|
|
secgroup_kubernetes:
|
|
type: "OS::Neutron::SecurityGroup"
|
|
properties:
|
|
rules:
|
|
- protocol: tcp
|
|
port_range_min: 7080
|
|
port_range_max: 7080
|
|
- protocol: tcp
|
|
port_range_min: 8080
|
|
port_range_max: 8080
|
|
- protocol: tcp
|
|
port_range_min: 4001
|
|
port_range_max: 4001
|
|
- protocol: tcp
|
|
port_range_min: 7001
|
|
port_range_max: 7001
|
|
|
|
######################################################################
|
|
#
|
|
# software configs. these are components that are combined into
|
|
# a multipart MIME user-data archive.
|
|
#
|
|
|
|
write_heat_params:
|
|
type: "OS::Heat::SoftwareConfig"
|
|
properties:
|
|
group: ungrouped
|
|
config:
|
|
str_replace:
|
|
template: {get_file: fragments/write-heat-params-master.yaml}
|
|
params:
|
|
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
|
|
"$WAIT_HANDLE": {get_resource: master_wait_handle}
|
|
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
|
|
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
|
|
"$FLANNEL_USE_VXLAN": {get_param: flannel_use_vxlan}
|
|
"$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr}
|
|
|
|
configure_kubernetes:
|
|
type: "OS::Heat::SoftwareConfig"
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/configure-kubernetes-master.sh}
|
|
|
|
write_flannel_config:
|
|
type: "OS::Heat::SoftwareConfig"
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/write-flannel-config.sh}
|
|
|
|
flannel_config_service:
|
|
type: "OS::Heat::SoftwareConfig"
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/flannel-config.service.yaml}
|
|
|
|
enable_services:
|
|
type: "OS::Heat::SoftwareConfig"
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/enable-services-master.sh}
|
|
|
|
kube_user:
|
|
type: "OS::Heat::SoftwareConfig"
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/kube-user.yaml}
|
|
|
|
kube_examples:
|
|
type: "OS::Heat::SoftwareConfig"
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/kube-examples.yaml}
|
|
|
|
cfn_signal:
|
|
type: "OS::Heat::SoftwareConfig"
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/cfn-signal.sh}
|
|
|
|
disable_selinux:
|
|
type: "OS::Heat::SoftwareConfig"
|
|
properties:
|
|
group: ungrouped
|
|
config: {get_file: fragments/disable-selinux.sh}
|
|
|
|
kube_master_init:
|
|
type: "OS::Heat::MultipartMime"
|
|
properties:
|
|
parts:
|
|
- config: {get_resource: disable_selinux}
|
|
- config: {get_resource: write_heat_params}
|
|
- config: {get_resource: kube_user}
|
|
- config: {get_resource: configure_kubernetes}
|
|
- config: {get_resource: enable_services}
|
|
- config: {get_resource: write_flannel_config}
|
|
- config: {get_resource: flannel_config_service}
|
|
- config: {get_resource: kube_examples}
|
|
- config: {get_resource: cfn_signal}
|
|
|
|
######################################################################
|
|
#
|
|
# databases server. this sets up a Kubernetes server
|
|
#
|
|
kube_master:
|
|
type: "OS::Nova::Server"
|
|
depends_on:
|
|
- extrouter_inside
|
|
properties:
|
|
image:
|
|
get_param: server_image
|
|
flavor:
|
|
get_param: master_flavor
|
|
key_name:
|
|
get_param: ssh_key_name
|
|
user_data_format: RAW
|
|
user_data: {get_resource: kube_master_init}
|
|
networks:
|
|
- port:
|
|
get_resource: kube_master_eth0
|
|
|
|
kube_master_eth0:
|
|
type: "OS::Neutron::Port"
|
|
properties:
|
|
network_id:
|
|
get_resource: fixed_network
|
|
security_groups:
|
|
- get_resource: secgroup_base
|
|
- get_resource: secgroup_kubernetes
|
|
fixed_ips:
|
|
- subnet_id:
|
|
get_resource: fixed_subnet
|
|
replacement_policy: AUTO
|
|
|
|
kube_master_floating:
|
|
type: "OS::Neutron::FloatingIP"
|
|
depends_on:
|
|
- extrouter_inside
|
|
properties:
|
|
floating_network_id:
|
|
get_param: external_network_id
|
|
port_id:
|
|
get_resource: kube_master_eth0
|
|
|
|
kube_minions:
|
|
type: "OS::Heat::ResourceGroup"
|
|
depends_on:
|
|
- extrouter_inside
|
|
- master_wait_condition
|
|
properties:
|
|
count: {get_param: number_of_minions}
|
|
resource_def:
|
|
type: kubenode.yaml
|
|
properties:
|
|
ssh_key_name: {get_param: ssh_key_name}
|
|
server_image: {get_param: server_image}
|
|
server_flavor: {get_param: server_flavor}
|
|
fixed_network_id: {get_resource: fixed_network}
|
|
fixed_subnet_id: {get_resource: fixed_subnet}
|
|
kube_master_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
|
|
external_network_id: {get_param: external_network_id}
|
|
kube_allow_priv: {get_param: kube_allow_priv}
|
|
docker_volume_size: {get_param: docker_volume_size}
|
|
|
|
outputs:
|
|
|
|
kube_master:
|
|
value: {get_attr: [kube_master_floating, floating_ip_address]}
|
|
|
|
kube_minions:
|
|
value: {get_attr: [kube_minions, kube_node_ip]}
|
|
|
|
kube_minions_external:
|
|
value: {get_attr: [kube_minions, kube_node_external_ip]}
|
|
|