From b1c36cdd31b8080b24b99d783bc3e9151255a5bc Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Tue, 30 Dec 2014 22:47:10 -0500 Subject: [PATCH] refactor config using software config support this splits the giant user-data script into smaller modules. this makes things easier to manage and also permits us to use cloud-config scripts in addition to shell scripts. --- beaker.yaml | 7 + fragments/add-centos-to-docker.sh | 12 ++ fragments/cfn-signal.sh | 11 ++ fragments/configure-flannel.sh | 18 +++ fragments/configure-kubernetes-master.sh | 16 ++ fragments/configure-kubernetes-minion.sh | 32 ++++ fragments/docker.service.yaml | 29 ++++ fragments/enable-services-master.sh | 9 ++ fragments/enable-services-minion.sh | 15 ++ fragments/kube-examples.yaml | 32 ++++ fragments/wait-for-flanneld.yaml | 33 +++++ fragments/write-heat-params-master.yaml | 12 ++ fragments/write-heat-params.yaml | 11 ++ kubecluster.yaml | 110 +++++++------- kubenode.yaml | 179 ++++++++--------------- 15 files changed, 357 insertions(+), 169 deletions(-) create mode 100644 beaker.yaml create mode 100644 fragments/add-centos-to-docker.sh create mode 100644 fragments/cfn-signal.sh create mode 100644 fragments/configure-flannel.sh create mode 100644 fragments/configure-kubernetes-master.sh create mode 100644 fragments/configure-kubernetes-minion.sh create mode 100644 fragments/docker.service.yaml create mode 100644 fragments/enable-services-master.sh create mode 100644 fragments/enable-services-minion.sh create mode 100644 fragments/kube-examples.yaml create mode 100644 fragments/wait-for-flanneld.yaml create mode 100644 fragments/write-heat-params-master.yaml create mode 100644 fragments/write-heat-params.yaml diff --git a/beaker.yaml b/beaker.yaml new file mode 100644 index 0000000..6c10fe5 --- /dev/null +++ b/beaker.yaml @@ -0,0 +1,7 @@ +parameters: + ssh_key_name: lars_redhat + server_image: fedora-21 + server_flavor: m1.medium + external_network_id: 59bcbd61-f5ed-4c77-8b60-b7a004ed40b3 + dns_nameserver: 10.16.36.29 + fixed_network_cidr: 192.168.113.0/24 diff --git a/fragments/add-centos-to-docker.sh b/fragments/add-centos-to-docker.sh new file mode 100644 index 0000000..da2caa0 --- /dev/null +++ b/fragments/add-centos-to-docker.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +# Under atomic, we need to make sure the 'docker' group exists in +# /etc/group (because /lib/group cannot be modified by usermod). +if ! grep -q docker /etc/group; then + grep docker /lib/group >> /etc/group +fi + +# make centos user a member of the docker group +# (so you can run docker commands as the centos user) +usermod -G docker centos + diff --git a/fragments/cfn-signal.sh b/fragments/cfn-signal.sh new file mode 100644 index 0000000..346f3ad --- /dev/null +++ b/fragments/cfn-signal.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. /etc/sysconfig/heat-params + +echo notifying heat +curl -sf -X PUT -H 'Content-Type: application/json' \ + --data-binary '{"Status": "SUCCESS", + "Reason": "Setup complete", + "Data": "OK", "UniqueId": "00000"}' \ + "$WAIT_HANDLE" + diff --git a/fragments/configure-flannel.sh b/fragments/configure-flannel.sh new file mode 100644 index 0000000..89dc8e4 --- /dev/null +++ b/fragments/configure-flannel.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +. /etc/sysconfig/heat-params + +# wait for etcd to become active (we will need it to push the flanneld config) +while ! curl -sf http://localhost:4001/v2/keys/; do + echo "waiting for etcd" + sleep 1 +done + +# put the flannel config in etcd +echo creating flanneld config in etcd +curl -sf -L http://localhost:4001/v2/keys/coreos.com/network/config \ + -X PUT -d value='{ + "Network": "'"$FLANNEL_NETWORK_CIDR"'", + "Subnetlen": '"$FLANNEL_NETWORK_SUBNETLEN"'}' + + diff --git a/fragments/configure-kubernetes-master.sh b/fragments/configure-kubernetes-master.sh new file mode 100644 index 0000000..257828e --- /dev/null +++ b/fragments/configure-kubernetes-master.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. /etc/sysconfig/heat-params + +sed -i ' + /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/ +' /etc/kubernetes/config + +sed -i ' + /^KUBE_API_ADDRESS=/ s/=.*/="--address=0.0.0.0"/ +' /etc/kubernetes/apiserver + +sed -i ' + /^KUBELET_ADDRESSES=/ s/=.*/="--machines='"$MINION_ADDRESSES"'"/ +' /etc/kubernetes/controller-manager + diff --git a/fragments/configure-kubernetes-minion.sh b/fragments/configure-kubernetes-minion.sh new file mode 100644 index 0000000..20ec772 --- /dev/null +++ b/fragments/configure-kubernetes-minion.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +. /etc/sysconfig/heat-params + +myip=$(ip addr show eth0 | +awk '$1 == "inet" {print $2}' | cut -f1 -d/) +myip_last_octet=${myip##*.} + +sed -i ' +/^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/ +/^KUBE_ETCD_SERVERS=/ s|=.*|="--etcd_servers=http://'"$KUBE_MASTER_IP"':4001"| +' /etc/kubernetes/config + +sed -i ' +' /etc/kubernetes/config + +sed -i ' +/^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/ +/^KUBELET_HOSTNAME=/ s/=.*/="--hostname_override='"$myip"'"/ +' /etc/kubernetes/kubelet + +sed -i ' +/^KUBE_MASTER=/ s/=.*/="--master='"$KUBE_MASTER_IP"':8080"/ +' /etc/kubernetes/apiserver + +sed -i ' +/^FLANNEL_ETCD=/ s|=.*|="http://'"$KUBE_MASTER_IP"':4001"| +' /etc/sysconfig/flanneld + +cat >> /etc/environment < whether or not kubernetes should permit privileged containers. @@ -146,6 +146,61 @@ resources: port_range_min: 7001 port_range_max: 7001 + write_heat_params: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/write-heat-params-master.yaml} + params: + "$MINION_ADDRESSES": {"Fn::Join": [",", {get_attr: [kube_minions, kube_node_ip]}]} + "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} + "$WAIT_HANDLE": {get_resource: master_wait_handle} + "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} + "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen} + + configure_kubernetes: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/configure-kubernetes-master.sh} + + configure_flannel: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/configure-flannel.sh} + + enable_services: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/enable-services-master.sh} + + kube_examples: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/kube-examples.yaml} + + cfn_signal: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/cfn-signal.sh} + + kube_master_init: + type: "OS::Heat::MultipartMime" + properties: + parts: + - config: {get_resource: write_heat_params} + - config: {get_resource: configure_kubernetes} + - config: {get_resource: enable_services} + - config: {get_resource: configure_flannel} + - config: {get_resource: kube_examples} + - config: {get_resource: cfn_signal} + ###################################################################### # # databases server. this sets up a Kubernetes server @@ -162,56 +217,7 @@ resources: key_name: get_param: ssh_key_name user_data_format: RAW - user_data: - str_replace: - template: | - #!/bin/sh - - sed -i ' - /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged=$ALLOW_PRIV"/ - ' /etc/kubernetes/config - - sed -i ' - /^KUBE_API_ADDRESS=/ s/=.*/="--address=0.0.0.0"/ - ' /etc/kubernetes/apiserver - - sed -i ' - /^KUBELET_ADDRESSES=/ s/=.*/="--machines=$MINION_ADDRESSES"/ - ' /etc/kubernetes/controller-manager - - echo starting services - for service in etcd kube-apiserver kube-scheduler kube-controller-manager; do - systemctl enable $service - systemctl start $service - done - - # wait for etcd to become active (we will need it to push the flanneld config) - while ! curl -sf http://localhost:4001/v2/keys/; do - echo "waiting for etcd" - sleep 1 - done - - # put the flannel config in etcd - echo creating flanneld config in etcd - curl -sf -L http://localhost:4001/v2/keys/coreos.com/network/config \ - -X PUT -d value='{ - "Network": "$FLANNEL_NETWORK_CIDR", - "Subnetlen": $FLANNEL_NETWORK_SUBNETLEN}' - - echo notifying heat - curl -sf -X PUT -H 'Content-Type: application/json' \ - --data-binary '{"Status": "SUCCESS", - "Reason": "Setup complete", - "Data": "OK", "UniqueId": "00000"}' \ - "$WAIT_HANDLE" - params: - # NB: For this to work you need a version of Heat that - # includes https://review.openstack.org/#/c/121139/ - "$MINION_ADDRESSES": {"Fn::Join": [",", {get_attr: [kube_minions, kube_node_ip]}]} - "$ALLOW_PRIV": {get_param: allow_priv} - "$WAIT_HANDLE": {get_resource: master_wait_handle} - "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} - "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen} + user_data: {get_resource: kube_master_init} networks: - port: get_resource: kube_master_eth0 @@ -254,7 +260,7 @@ resources: fixed_subnet_id: {get_resource: fixed_subnet} kube_master_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} external_network_id: {get_param: external_network_id} - allow_priv: {get_param: allow_priv} + kube_allow_priv: {get_param: kube_allow_priv} outputs: diff --git a/kubenode.yaml b/kubenode.yaml index 20ad0a3..6db2fbd 100644 --- a/kubenode.yaml +++ b/kubenode.yaml @@ -26,7 +26,7 @@ parameters: type: string description: uuid of a network to use for floating ip addresses - allow_priv: + kube_allow_priv: type: string description: > whether or not kubernetes should permit privileged containers. @@ -68,6 +68,66 @@ resources: - protocol: tcp - protocol: udp + write_heat_params: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: + str_replace: + template: {get_file: fragments/write-heat-params.yaml} + params: + "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} + "$KUBE_MASTER_IP": {get_param: kube_master_ip} + "$WAIT_HANDLE": {get_resource: node_wait_handle} + + add_centos_to_docker: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/add-centos-to-docker.sh} + + configure_kubernetes_minion: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/configure-kubernetes-minion.sh} + + docker_service: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/docker.service.yaml} + + wait_for_flanneld: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/wait-for-flanneld.yaml} + + enable_services: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/enable-services-minion.sh} + + cfn_signal: + type: "OS::Heat::SoftwareConfig" + properties: + group: ungrouped + config: {get_file: fragments/cfn-signal.sh} + + kube_node_init: + type: "OS::Heat::MultipartMime" + properties: + parts: + - config: {get_resource: write_heat_params} + - config: {get_resource: add_centos_to_docker} + - config: {get_resource: configure_kubernetes_minion} + - config: {get_resource: docker_service} + - config: {get_resource: wait_for_flanneld} + - config: {get_resource: enable_services} + - config: {get_resource: cfn_signal} + kube_node: type: "OS::Nova::Server" properties: @@ -78,122 +138,7 @@ resources: key_name: get_param: ssh_key_name user_data_format: RAW - user_data: - str_replace: - template: | - #!/bin/sh - - myip=$(ip addr show eth0 | - awk '$1 == "inet" {print $2}' | cut -f1 -d/) - myip_last_octet=${myip##*.} - - sed -i ' - /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged=$ALLOW_PRIV"/ - ' /etc/kubernetes/config - - sed -i '/^KUBE_ETCD_SERVERS=/ s|=.*|="--etcd_servers=http://$KUBE_MASTER_IP:4001"|' \ - /etc/kubernetes/config - - sed -i ' - /^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/ - /^KUBELET_HOSTNAME=/ s/=.*/="--hostname_override='"$myip"'"/ - ' /etc/kubernetes/kubelet - - sed -i ' - /^KUBE_MASTER=/ s/=.*/="--master=$KUBE_MASTER_IP:8080"/ - ' /etc/kubernetes/apiserver - - sed -i ' - /^FLANNEL_ETCD=/ s|=.*|="http://$KUBE_MASTER_IP:4001"| - ' /etc/sysconfig/flanneld - - cat >> /etc/environment <> /etc/group - fi - usermod -G docker centos - - cat > /usr/local/bin/flanneld-waiter <<'EOF' - #!/bin/sh - - while ! [ -f /run/flannel/subnet.env ]; do - echo "waiting for flanneld" - sleep 1 - done - - echo flanneld is active - - exit 0 - EOF - - chmod 755 /usr/local/bin/flanneld-waiter - - cat > /etc/systemd/system/flanneld-waiter.service <<'EOF' - [Unit] - Description=Wait for flanneld to provide subnet/mtu information - After=network.target flanneld.service - Requires=flanneld.service - - [Service] - Type=oneshot - ExecStart=/usr/local/bin/flanneld-waiter - - [Install] - WantedBy=multi-user.target - EOF - - cat > /etc/systemd/system/docker.service <<'EOF' - [Unit] - Description=Docker Application Container Engine - Documentation=http://docs.docker.com - After=network.target docker.socket flanneld-waiter.service - Requires=docker.socket flanneld-waiter.service - - [Service] - Type=notify - EnvironmentFile=-/etc/sysconfig/docker - EnvironmentFile=-/etc/sysconfig/docker-storage - EnvironmentFile=-/run/flannel/subnet.env - ExecStart=/usr/bin/docker -d -H fd:// --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU $OPTIONS $DOCKER_STORAGE_OPTIONS - Restart=on-failure - LimitNOFILE=1048576 - LimitNPROC=1048576 - - [Install] - WantedBy=multi-user.target - EOF - - echo reloading systemd - systemctl daemon-reload - - # docker is already enabled and possibly running on centos atomic host - # so we need to stop it first and delete the docker0 bridge (which will - # be re-created using the flannel-provided subnet). - echo stopping docker - systemctl stop docker - ip link del docker0 - - echo starting services - for service in flanneld-waiter flanneld docker.socket kubelet kube-proxy; do - systemctl enable $service - systemctl --no-block start $service - done - - echo notifying heat - curl -sf -X PUT -H 'Content-Type: application/json' \ - --data-binary '{"Status": "SUCCESS", - "Reason": "Setup complete", - "Data": "OK", "UniqueId": "00000"}' \ - "$WAIT_HANDLE" - params: - "$ALLOW_PRIV": {get_param: allow_priv} - "$KUBE_MASTER_IP": {get_param: kube_master_ip} - "$WAIT_HANDLE": {get_resource: node_wait_handle} + user_data: {get_resource: kube_node_init} networks: - port: get_resource: kube_node_eth0