stackforge/heat-coe-templates
Code Issues Proposed changes

refactor config using software config support

Browse Source 
this splits the giant user-data script into smaller modules.  this
makes things easier to manage and also permits us to use cloud-config
scripts in addition to shell scripts.
This commit is contained in:
Lars Kellogg-Stedman 2014-12-30 22:47:10 -05:00
parent 0d981def48
commit b1c36cdd31
15 changed files with 357 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
beaker.yaml Normal file
View File

@ -0,0 +1,7 @@
parameters:
ssh_key_name: lars_redhat
server_image: fedora-21
server_flavor: m1.medium
external_network_id: 59bcbd61-f5ed-4c77-8b60-b7a004ed40b3
dns_nameserver: 10.16.36.29
fixed_network_cidr: 192.168.113.0/24

12
fragments/add-centos-to-docker.sh Normal file
View File

@ -0,0 +1,12 @@
#!/bin/sh
# Under atomic, we need to make sure the 'docker' group exists in
# /etc/group (because /lib/group cannot be modified by usermod).
if ! grep -q docker /etc/group; then
grep docker /lib/group >> /etc/group
fi
# make centos user a member of the docker group
# (so you can run docker commands as the centos user)
usermod -G docker centos

11
fragments/cfn-signal.sh Normal file
View File

@ -0,0 +1,11 @@
#!/bin/sh
. /etc/sysconfig/heat-params
echo notifying heat
curl -sf -X PUT -H 'Content-Type: application/json' \
--data-binary '{"Status": "SUCCESS",
"Reason": "Setup complete",
"Data": "OK", "UniqueId": "00000"}' \
"$WAIT_HANDLE"

18
fragments/configure-flannel.sh Normal file
View File

@ -0,0 +1,18 @@
#!/bin/sh
. /etc/sysconfig/heat-params
# wait for etcd to become active (we will need it to push the flanneld config)
while ! curl -sf http://localhost:4001/v2/keys/; do
echo "waiting for etcd"
sleep 1
done
# put the flannel config in etcd
echo creating flanneld config in etcd
curl -sf -L http://localhost:4001/v2/keys/coreos.com/network/config \
-X PUT -d value='{
"Network": "'"$FLANNEL_NETWORK_CIDR"'",
"Subnetlen": '"$FLANNEL_NETWORK_SUBNETLEN"'}'

16
fragments/configure-kubernetes-master.sh Normal file
View File

@ -0,0 +1,16 @@
#!/bin/sh
. /etc/sysconfig/heat-params
sed -i '
/^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/
' /etc/kubernetes/config
sed -i '
/^KUBE_API_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
' /etc/kubernetes/apiserver
sed -i '
/^KUBELET_ADDRESSES=/ s/=.*/="--machines='"$MINION_ADDRESSES"'"/
' /etc/kubernetes/controller-manager

32
fragments/configure-kubernetes-minion.sh Normal file
View File

@ -0,0 +1,32 @@
#!/bin/sh
. /etc/sysconfig/heat-params
myip=$(ip addr show eth0 |
awk '$1 == "inet" {print $2}' | cut -f1 -d/)
myip_last_octet=${myip##*.}
sed -i '
/^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/
/^KUBE_ETCD_SERVERS=/ s|=.*|="--etcd_servers=http://'"$KUBE_MASTER_IP"':4001"|
' /etc/kubernetes/config
sed -i '
' /etc/kubernetes/config
sed -i '
/^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
/^KUBELET_HOSTNAME=/ s/=.*/="--hostname_override='"$myip"'"/
' /etc/kubernetes/kubelet
sed -i '
/^KUBE_MASTER=/ s/=.*/="--master='"$KUBE_MASTER_IP"':8080"/
' /etc/kubernetes/apiserver
sed -i '
/^FLANNEL_ETCD=/ s|=.*|="http://'"$KUBE_MASTER_IP"':4001"|
' /etc/sysconfig/flanneld
cat >> /etc/environment <<EOF
KUBERNETES_MASTER=http://$KUBE_MASTER_IP:8080
EOF

29
fragments/docker.service.yaml Normal file
View File

@ -0,0 +1,29 @@
#cloud-config
merge_how: dict(recurse_array)+list(append)
write_files:
- path: /etc/systemd/system/docker.service
owner: "root:root"
permissions: "0644"
content: |
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.com
After=network.target docker.socket wait-for-flanneld.service
Requires=docker.socket wait-for-flanneld.service
[Service]
Type=notify
EnvironmentFile=-/etc/sysconfig/docker
EnvironmentFile=-/etc/sysconfig/docker-storage
# This EnvironmentFile is required.
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/docker -d -H fd:// --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU $OPTIONS $DOCKER_STORAGE_OPTIONS
Restart=on-failure
LimitNOFILE=1048576
LimitNPROC=1048576
[Install]
WantedBy=multi-user.target

9
fragments/enable-services-master.sh Normal file
View File

@ -0,0 +1,9 @@
#!/bin/sh
echo starting services
for service in etcd kube-apiserver kube-scheduler kube-controller-manager; do
systemctl enable $service
systemctl --no-block start $service
done

15
fragments/enable-services-minion.sh Normal file
View File

@ -0,0 +1,15 @@
#!/bin/sh
# docker is already enabled and possibly running on centos atomic host
# so we need to stop it first and delete the docker0 bridge (which will
# be re-created using the flannel-provided subnet).
echo stopping docker
systemctl stop docker
ip link del docker0
echo starting services
for service in wait-for-flanneld flanneld docker.socket docker kubelet kube-proxy; do
systemctl enable $service
systemctl --no-block start $service
done

32
fragments/kube-examples.yaml Normal file
View File

@ -0,0 +1,32 @@
#cloud-config
merge_how: dict(recurse_array)+list(append)
write_files:
- path: /etc/kubernetes/examples/web.pod
owner: "root:root"
permissions: "0644"
content: |
kind: Pod
apiVersion: v1beta1
labels:
name: web
desiredState:
manifest:
version: v1beta1
id: web
containers:
- name: web
image: larsks/onehttpd
command: [/onehttpd, -p, 80, /data]
ports:
- containerPort: 80
- path: /etc/kubernetes/examples/web.service
owner: "root:root"
permissions: "0644"
content: |
kind: Service
apiVersion: v1beta1
id: web
port: 8000
selector:
name: web
containerPort: 80

33
fragments/wait-for-flanneld.yaml Normal file
View File

@ -0,0 +1,33 @@
#cloud-config
merge_how: dict(recurse_array)+list(append)
write_files:
- path: /usr/local/bin/wait-for-flanneld
owner: "root:root"
permissions: "0755"
content: |
#!/bin/sh
while ! [ -f /run/flannel/subnet.env ]; do
echo "waiting for flanneld"
sleep 1
done
echo flanneld is active
exit 0
- path: /etc/systemd/system/wait-for-flanneld.service
owner: "root:root"
permissions: "0644"
content: |
[Unit]
Description=Wait for flanneld to provide subnet/mtu information
After=network.target flanneld.service
Requires=flanneld.service
[Service]
Type=oneshot
ExecStart=/usr/local/bin/wait-for-flanneld
[Install]
WantedBy=multi-user.target

12
fragments/write-heat-params-master.yaml Normal file
View File

@ -0,0 +1,12 @@
#cloud-config
merge_how: dict(recurse_array)+list(append)
write_files:
- path: /etc/sysconfig/heat-params
owner: "root:root"
permissions: "0644"
content: |
MINION_ADDRESSES="$MINION_ADDRESSES"
KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
WAIT_HANDLE="$WAIT_HANDLE"
FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
FLANNEL_NETWORK_SUBNETLEN="$FLANNEL_NETWORK_SUBNETLEN"

11
fragments/write-heat-params.yaml Normal file
View File

@ -0,0 +1,11 @@
#cloud-config
merge_how: dict(recurse_array)+list(append)
write_files:
- path: /etc/sysconfig/heat-params
owner: "root:root"
permissions: "0644"
content: |
KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
KUBE_MASTER_IP="$KUBE_MASTER_IP"
WAIT_HANDLE="$WAIT_HANDLE"

110
kubecluster.yaml
View File

@ -56,7 +56,7 @@ parameters:
description: size of subnet assigned to each minion
default: 24
allow_priv:
kube_allow_priv:
type: string
description: >
whether or not kubernetes should permit privileged containers.
@ -146,6 +146,61 @@ resources:
port_range_min: 7001
port_range_max: 7001
write_heat_params:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config:
str_replace:
template: {get_file: fragments/write-heat-params-master.yaml}
params:
"$MINION_ADDRESSES": {"Fn::Join": [",", {get_attr: [kube_minions, kube_node_ip]}]}
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
"$WAIT_HANDLE": {get_resource: master_wait_handle}
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
configure_kubernetes:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/configure-kubernetes-master.sh}
configure_flannel:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/configure-flannel.sh}
enable_services:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/enable-services-master.sh}
kube_examples:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/kube-examples.yaml}
cfn_signal:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/cfn-signal.sh}
kube_master_init:
type: "OS::Heat::MultipartMime"
properties:
parts:
- config: {get_resource: write_heat_params}
- config: {get_resource: configure_kubernetes}
- config: {get_resource: enable_services}
- config: {get_resource: configure_flannel}
- config: {get_resource: kube_examples}
- config: {get_resource: cfn_signal}
######################################################################
#
# databases server. this sets up a Kubernetes server
@ -162,56 +217,7 @@ resources:
key_name:
get_param: ssh_key_name
user_data_format: RAW
user_data:
str_replace:
template: |
#!/bin/sh
sed -i '
/^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged=$ALLOW_PRIV"/
' /etc/kubernetes/config
sed -i '
/^KUBE_API_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
' /etc/kubernetes/apiserver
sed -i '
/^KUBELET_ADDRESSES=/ s/=.*/="--machines=$MINION_ADDRESSES"/
' /etc/kubernetes/controller-manager
echo starting services
for service in etcd kube-apiserver kube-scheduler kube-controller-manager; do
systemctl enable $service
systemctl start $service
done
# wait for etcd to become active (we will need it to push the flanneld config)
while ! curl -sf http://localhost:4001/v2/keys/; do
echo "waiting for etcd"
sleep 1
done
# put the flannel config in etcd
echo creating flanneld config in etcd
curl -sf -L http://localhost:4001/v2/keys/coreos.com/network/config \
-X PUT -d value='{
"Network": "$FLANNEL_NETWORK_CIDR",
"Subnetlen": $FLANNEL_NETWORK_SUBNETLEN}'
echo notifying heat
curl -sf -X PUT -H 'Content-Type: application/json' \
--data-binary '{"Status": "SUCCESS",
"Reason": "Setup complete",
"Data": "OK", "UniqueId": "00000"}' \
"$WAIT_HANDLE"
params:
# NB: For this to work you need a version of Heat that
# includes https://review.openstack.org/#/c/121139/
"$MINION_ADDRESSES": {"Fn::Join": [",", {get_attr: [kube_minions, kube_node_ip]}]}
"$ALLOW_PRIV": {get_param: allow_priv}
"$WAIT_HANDLE": {get_resource: master_wait_handle}
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
user_data: {get_resource: kube_master_init}
networks:
- port:
get_resource: kube_master_eth0
@ -254,7 +260,7 @@ resources:
fixed_subnet_id: {get_resource: fixed_subnet}
kube_master_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
external_network_id: {get_param: external_network_id}
allow_priv: {get_param: allow_priv}
kube_allow_priv: {get_param: kube_allow_priv}
outputs:

179
kubenode.yaml
View File

@ -26,7 +26,7 @@ parameters:
type: string
description: uuid of a network to use for floating ip addresses
allow_priv:
kube_allow_priv:
type: string
description: >
whether or not kubernetes should permit privileged containers.
@ -68,6 +68,66 @@ resources:
- protocol: tcp
- protocol: udp
write_heat_params:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config:
str_replace:
template: {get_file: fragments/write-heat-params.yaml}
params:
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
"$KUBE_MASTER_IP": {get_param: kube_master_ip}
"$WAIT_HANDLE": {get_resource: node_wait_handle}
add_centos_to_docker:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/add-centos-to-docker.sh}
configure_kubernetes_minion:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/configure-kubernetes-minion.sh}
docker_service:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/docker.service.yaml}
wait_for_flanneld:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/wait-for-flanneld.yaml}
enable_services:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/enable-services-minion.sh}
cfn_signal:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/cfn-signal.sh}
kube_node_init:
type: "OS::Heat::MultipartMime"
properties:
parts:
- config: {get_resource: write_heat_params}
- config: {get_resource: add_centos_to_docker}
- config: {get_resource: configure_kubernetes_minion}
- config: {get_resource: docker_service}
- config: {get_resource: wait_for_flanneld}
- config: {get_resource: enable_services}
- config: {get_resource: cfn_signal}
kube_node:
type: "OS::Nova::Server"
properties:
@ -78,122 +138,7 @@ resources:
key_name:
get_param: ssh_key_name
user_data_format: RAW
user_data:
str_replace:
template: |
#!/bin/sh
myip=$(ip addr show eth0 |
awk '$1 == "inet" {print $2}' | cut -f1 -d/)
myip_last_octet=${myip##*.}
sed -i '
/^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged=$ALLOW_PRIV"/
' /etc/kubernetes/config
sed -i '/^KUBE_ETCD_SERVERS=/ s|=.*|="--etcd_servers=http://$KUBE_MASTER_IP:4001"|' \
/etc/kubernetes/config
sed -i '
/^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
/^KUBELET_HOSTNAME=/ s/=.*/="--hostname_override='"$myip"'"/
' /etc/kubernetes/kubelet
sed -i '
/^KUBE_MASTER=/ s/=.*/="--master=$KUBE_MASTER_IP:8080"/
' /etc/kubernetes/apiserver
sed -i '
/^FLANNEL_ETCD=/ s|=.*|="http://$KUBE_MASTER_IP:4001"|
' /etc/sysconfig/flanneld
cat >> /etc/environment <<EOF
KUBERNETES_MASTER=http://$KUBE_MASTER_IP:8080
EOF
# make centos user a member of the docker group
# (so you can run docker commands as the centos user)
if ! grep -q docker /etc/group; then
grep docker /lib/group >> /etc/group
fi
usermod -G docker centos
cat > /usr/local/bin/flanneld-waiter <<'EOF'
#!/bin/sh
while ! [ -f /run/flannel/subnet.env ]; do
echo "waiting for flanneld"
sleep 1
done
echo flanneld is active
exit 0
EOF
chmod 755 /usr/local/bin/flanneld-waiter
cat > /etc/systemd/system/flanneld-waiter.service <<'EOF'
[Unit]
Description=Wait for flanneld to provide subnet/mtu information
After=network.target flanneld.service
Requires=flanneld.service
[Service]
Type=oneshot
ExecStart=/usr/local/bin/flanneld-waiter
[Install]
WantedBy=multi-user.target
EOF
cat > /etc/systemd/system/docker.service <<'EOF'
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.com
After=network.target docker.socket flanneld-waiter.service
Requires=docker.socket flanneld-waiter.service
[Service]
Type=notify
EnvironmentFile=-/etc/sysconfig/docker
EnvironmentFile=-/etc/sysconfig/docker-storage
EnvironmentFile=-/run/flannel/subnet.env
ExecStart=/usr/bin/docker -d -H fd:// --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU $OPTIONS $DOCKER_STORAGE_OPTIONS
Restart=on-failure
LimitNOFILE=1048576
LimitNPROC=1048576
[Install]
WantedBy=multi-user.target
EOF
echo reloading systemd
systemctl daemon-reload
# docker is already enabled and possibly running on centos atomic host
# so we need to stop it first and delete the docker0 bridge (which will
# be re-created using the flannel-provided subnet).
echo stopping docker
systemctl stop docker
ip link del docker0
echo starting services
for service in flanneld-waiter flanneld docker.socket kubelet kube-proxy; do
systemctl enable $service
systemctl --no-block start $service
done
echo notifying heat
curl -sf -X PUT -H 'Content-Type: application/json' \
--data-binary '{"Status": "SUCCESS",
"Reason": "Setup complete",
"Data": "OK", "UniqueId": "00000"}' \
"$WAIT_HANDLE"
params:
"$ALLOW_PRIV": {get_param: allow_priv}
"$KUBE_MASTER_IP": {get_param: kube_master_ip}
"$WAIT_HANDLE": {get_resource: node_wait_handle}
user_data: {get_resource: kube_node_init}
networks:
- port:
get_resource: kube_node_eth0