refactor config using software config support

this splits the giant user-data script into smaller modules.  this
makes things easier to manage and also permits us to use cloud-config
scripts in addition to shell scripts.

beaker.yaml

@@ -0,0 +1,7 @@
+parameters:
+  ssh_key_name: lars_redhat
+  server_image: fedora-21
+  server_flavor: m1.medium
+  external_network_id: 59bcbd61-f5ed-4c77-8b60-b7a004ed40b3
+  dns_nameserver: 10.16.36.29
+  fixed_network_cidr: 192.168.113.0/24

fragments/add-centos-to-docker.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Under atomic, we need to make sure the 'docker' group exists in 
+# /etc/group (because /lib/group cannot be modified by usermod).
+if ! grep -q docker /etc/group; then
+	grep docker /lib/group >> /etc/group
+fi
+
+# make centos user a member of the docker group
+# (so you can run docker commands as the centos user)
+usermod -G docker centos
+

fragments/cfn-signal.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+. /etc/sysconfig/heat-params
+
+echo notifying heat
+curl -sf -X PUT -H 'Content-Type: application/json' \
+	--data-binary '{"Status": "SUCCESS",
+	"Reason": "Setup complete",
+	"Data": "OK", "UniqueId": "00000"}' \
+	"$WAIT_HANDLE"
+

fragments/configure-flannel.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. /etc/sysconfig/heat-params
+
+# wait for etcd to become active (we will need it to push the flanneld config)
+while ! curl -sf http://localhost:4001/v2/keys/; do
+  echo "waiting for etcd"
+  sleep 1
+done
+
+# put the flannel config in etcd
+echo creating flanneld config in etcd
+curl -sf -L http://localhost:4001/v2/keys/coreos.com/network/config \
+  -X PUT -d value='{
+    "Network": "'"$FLANNEL_NETWORK_CIDR"'",
+    "Subnetlen": '"$FLANNEL_NETWORK_SUBNETLEN"'}'
+
+

fragments/configure-kubernetes-master.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+. /etc/sysconfig/heat-params
+
+sed -i '
+  /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/
+' /etc/kubernetes/config
+
+sed -i '
+  /^KUBE_API_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
+' /etc/kubernetes/apiserver
+
+sed -i '
+  /^KUBELET_ADDRESSES=/ s/=.*/="--machines='"$MINION_ADDRESSES"'"/
+' /etc/kubernetes/controller-manager
+

fragments/configure-kubernetes-minion.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+. /etc/sysconfig/heat-params
+
+myip=$(ip addr show eth0 |
+awk '$1 == "inet" {print $2}' | cut -f1 -d/)
+myip_last_octet=${myip##*.}
+
+sed -i '
+/^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/
+/^KUBE_ETCD_SERVERS=/ s|=.*|="--etcd_servers=http://'"$KUBE_MASTER_IP"':4001"|
+' /etc/kubernetes/config
+
+sed -i '
+' /etc/kubernetes/config
+
+sed -i '
+/^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
+/^KUBELET_HOSTNAME=/ s/=.*/="--hostname_override='"$myip"'"/
+' /etc/kubernetes/kubelet
+
+sed -i '
+/^KUBE_MASTER=/ s/=.*/="--master='"$KUBE_MASTER_IP"':8080"/
+' /etc/kubernetes/apiserver
+
+sed -i '
+/^FLANNEL_ETCD=/ s|=.*|="http://'"$KUBE_MASTER_IP"':4001"|
+' /etc/sysconfig/flanneld
+
+cat >> /etc/environment <<EOF
+KUBERNETES_MASTER=http://$KUBE_MASTER_IP:8080
+EOF

fragments/docker.service.yaml

@@ -0,0 +1,29 @@
+#cloud-config
+merge_how: dict(recurse_array)+list(append)
+write_files:
+  - path: /etc/systemd/system/docker.service
+    owner: "root:root"
+    permissions: "0644"
+    content: |
+      [Unit]
+      Description=Docker Application Container Engine
+      Documentation=http://docs.docker.com
+      After=network.target docker.socket wait-for-flanneld.service
+      Requires=docker.socket wait-for-flanneld.service
+
+      [Service]
+      Type=notify
+      EnvironmentFile=-/etc/sysconfig/docker
+      EnvironmentFile=-/etc/sysconfig/docker-storage
+      
+      # This EnvironmentFile is required.
+      EnvironmentFile=/run/flannel/subnet.env
+
+      ExecStart=/usr/bin/docker -d -H fd:// --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU $OPTIONS $DOCKER_STORAGE_OPTIONS
+      Restart=on-failure
+      LimitNOFILE=1048576
+      LimitNPROC=1048576
+
+      [Install]
+      WantedBy=multi-user.target
+

fragments/enable-services-master.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+echo starting services
+for service in etcd kube-apiserver kube-scheduler kube-controller-manager; do
+  systemctl enable $service
+  systemctl --no-block start $service
+done
+
+

fragments/enable-services-minion.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# docker is already enabled and possibly running on centos atomic host
+# so we need to stop it first and delete the docker0 bridge (which will
+# be re-created using the flannel-provided subnet).
+echo stopping docker
+systemctl stop docker
+ip link del docker0
+
+echo starting services
+for service in wait-for-flanneld flanneld docker.socket docker kubelet kube-proxy; do
+	systemctl enable $service
+	systemctl --no-block start $service
+done
+

fragments/kube-examples.yaml

@@ -0,0 +1,32 @@
+#cloud-config
+merge_how: dict(recurse_array)+list(append)
+write_files:
+  - path: /etc/kubernetes/examples/web.pod
+    owner: "root:root"
+    permissions: "0644"
+    content: |
+      kind: Pod
+      apiVersion: v1beta1
+      labels:
+        name: web
+      desiredState:
+        manifest:
+          version: v1beta1
+          id: web
+          containers:
+            - name: web
+              image: larsks/onehttpd
+              command: [/onehttpd, -p, 80, /data]
+              ports:
+                - containerPort: 80
+  - path: /etc/kubernetes/examples/web.service
+    owner: "root:root"
+    permissions: "0644"
+    content: |
+      kind: Service
+      apiVersion: v1beta1
+      id: web
+      port: 8000
+      selector:
+        name: web
+      containerPort: 80

fragments/wait-for-flanneld.yaml

@@ -0,0 +1,33 @@
+#cloud-config
+merge_how: dict(recurse_array)+list(append)
+write_files:
+  - path: /usr/local/bin/wait-for-flanneld
+    owner: "root:root"
+    permissions: "0755"
+    content: |
+      #!/bin/sh
+      
+      while ! [ -f /run/flannel/subnet.env ]; do
+        echo "waiting for flanneld"
+        sleep 1
+      done
+
+      echo flanneld is active
+
+      exit 0
+  - path: /etc/systemd/system/wait-for-flanneld.service
+    owner: "root:root"
+    permissions: "0644"
+    content: |
+      [Unit]
+      Description=Wait for flanneld to provide subnet/mtu information
+      After=network.target flanneld.service
+      Requires=flanneld.service
+
+      [Service]
+      Type=oneshot
+      ExecStart=/usr/local/bin/wait-for-flanneld
+
+      [Install]
+      WantedBy=multi-user.target
+

fragments/write-heat-params-master.yaml

@@ -0,0 +1,12 @@
+#cloud-config
+merge_how: dict(recurse_array)+list(append)
+write_files:
+  - path: /etc/sysconfig/heat-params
+    owner: "root:root"
+    permissions: "0644"
+    content: |
+      MINION_ADDRESSES="$MINION_ADDRESSES"
+      KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
+      WAIT_HANDLE="$WAIT_HANDLE"
+      FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
+      FLANNEL_NETWORK_SUBNETLEN="$FLANNEL_NETWORK_SUBNETLEN"

fragments/write-heat-params.yaml

@@ -0,0 +1,11 @@
+#cloud-config
+merge_how: dict(recurse_array)+list(append)
+write_files:
+  - path: /etc/sysconfig/heat-params
+    owner: "root:root"
+    permissions: "0644"
+    content: |
+      KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
+      KUBE_MASTER_IP="$KUBE_MASTER_IP"
+      WAIT_HANDLE="$WAIT_HANDLE"
+

kubecluster.yaml

@@ -56,7 +56,7 @@ parameters:
     description: size of subnet assigned to each minion
     default: 24
 
-  allow_priv:
+  kube_allow_priv:
     type: string
     description: >
       whether or not kubernetes should permit privileged containers.
@@ -146,6 +146,61 @@ resources:
           port_range_min: 7001
           port_range_max: 7001
 
+  write_heat_params:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config:
+        str_replace:
+          template: {get_file: fragments/write-heat-params-master.yaml}
+          params:
+            "$MINION_ADDRESSES": {"Fn::Join": [",", {get_attr: [kube_minions, kube_node_ip]}]}
+            "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
+            "$WAIT_HANDLE": {get_resource: master_wait_handle}
+            "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
+            "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
+
+  configure_kubernetes:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/configure-kubernetes-master.sh}
+
+  configure_flannel:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/configure-flannel.sh}
+
+  enable_services:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/enable-services-master.sh}
+
+  kube_examples:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/kube-examples.yaml}
+
+  cfn_signal:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/cfn-signal.sh}
+
+  kube_master_init:
+    type: "OS::Heat::MultipartMime"
+    properties:
+      parts:
+        - config: {get_resource: write_heat_params}
+        - config: {get_resource: configure_kubernetes}
+        - config: {get_resource: enable_services}
+        - config: {get_resource: configure_flannel}
+        - config: {get_resource: kube_examples}
+        - config: {get_resource: cfn_signal}
+
   ######################################################################
   #
   # databases server.  this sets up a Kubernetes server
@@ -162,56 +217,7 @@ resources:
       key_name:
         get_param: ssh_key_name
       user_data_format: RAW
-      user_data:
+      user_data: {get_resource: kube_master_init}
-        str_replace:
-          template: |
-            #!/bin/sh
-
-            sed -i '
-              /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged=$ALLOW_PRIV"/
-            ' /etc/kubernetes/config
-
-            sed -i '
-              /^KUBE_API_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
-            ' /etc/kubernetes/apiserver
-
-            sed -i '
-              /^KUBELET_ADDRESSES=/ s/=.*/="--machines=$MINION_ADDRESSES"/
-            ' /etc/kubernetes/controller-manager
-
-            echo starting services
-            for service in etcd kube-apiserver kube-scheduler kube-controller-manager; do
-              systemctl enable $service
-              systemctl start $service
-            done
-
-            # wait for etcd to become active (we will need it to push the flanneld config)
-            while ! curl -sf http://localhost:4001/v2/keys/; do
-              echo "waiting for etcd"
-              sleep 1
-            done
-
-            # put the flannel config in etcd
-            echo creating flanneld config in etcd
-            curl -sf -L http://localhost:4001/v2/keys/coreos.com/network/config \
-              -X PUT -d value='{
-                "Network": "$FLANNEL_NETWORK_CIDR",
-                "Subnetlen": $FLANNEL_NETWORK_SUBNETLEN}'
-
-            echo notifying heat
-            curl -sf -X PUT -H 'Content-Type: application/json' \
-              --data-binary '{"Status": "SUCCESS",
-                "Reason": "Setup complete",
-                "Data": "OK", "UniqueId": "00000"}' \
-              "$WAIT_HANDLE"
-          params:
-            # NB: For this to work you need a version of Heat that
-            # includes https://review.openstack.org/#/c/121139/
-            "$MINION_ADDRESSES": {"Fn::Join": [",", {get_attr: [kube_minions, kube_node_ip]}]}
-            "$ALLOW_PRIV": {get_param: allow_priv}
-            "$WAIT_HANDLE": {get_resource: master_wait_handle}
-            "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
-            "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
       networks:
         - port:
             get_resource: kube_master_eth0
@@ -254,7 +260,7 @@ resources:
           fixed_subnet_id: {get_resource: fixed_subnet}
           kube_master_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
           external_network_id: {get_param: external_network_id}
-          allow_priv: {get_param: allow_priv}
+          kube_allow_priv: {get_param: kube_allow_priv}
 
 outputs:
 

kubenode.yaml

@@ -26,7 +26,7 @@ parameters:
     type: string
     description: uuid of a network to use for floating ip addresses
 
-  allow_priv:
+  kube_allow_priv:
     type: string
     description: >
       whether or not kubernetes should permit privileged containers.
@@ -68,6 +68,66 @@ resources:
         - protocol: tcp
         - protocol: udp
 
+  write_heat_params:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config:
+        str_replace:
+          template: {get_file: fragments/write-heat-params.yaml}
+          params:
+            "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
+            "$KUBE_MASTER_IP": {get_param: kube_master_ip}
+            "$WAIT_HANDLE": {get_resource: node_wait_handle}
+
+  add_centos_to_docker:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/add-centos-to-docker.sh}
+
+  configure_kubernetes_minion:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/configure-kubernetes-minion.sh}
+
+  docker_service:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/docker.service.yaml}
+
+  wait_for_flanneld:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/wait-for-flanneld.yaml}
+
+  enable_services:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/enable-services-minion.sh}
+
+  cfn_signal:
+    type: "OS::Heat::SoftwareConfig"
+    properties:
+      group: ungrouped
+      config: {get_file: fragments/cfn-signal.sh}
+
+  kube_node_init:
+    type: "OS::Heat::MultipartMime"
+    properties:
+      parts:
+        - config: {get_resource: write_heat_params}
+        - config: {get_resource: add_centos_to_docker}
+        - config: {get_resource: configure_kubernetes_minion}
+        - config: {get_resource: docker_service}
+        - config: {get_resource: wait_for_flanneld}
+        - config: {get_resource: enable_services}
+        - config: {get_resource: cfn_signal}
+
   kube_node:
     type: "OS::Nova::Server"
     properties:
@@ -78,122 +138,7 @@ resources:
       key_name:
         get_param: ssh_key_name
       user_data_format: RAW
-      user_data:
+      user_data: {get_resource: kube_node_init}
-        str_replace:
-          template: |
-            #!/bin/sh
-
-            myip=$(ip addr show eth0 |
-              awk '$1 == "inet" {print $2}' | cut -f1 -d/)
-            myip_last_octet=${myip##*.}
-
-            sed -i '
-              /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged=$ALLOW_PRIV"/
-            ' /etc/kubernetes/config
-
-            sed -i '/^KUBE_ETCD_SERVERS=/ s|=.*|="--etcd_servers=http://$KUBE_MASTER_IP:4001"|' \
-              /etc/kubernetes/config
-
-            sed -i '
-              /^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
-              /^KUBELET_HOSTNAME=/ s/=.*/="--hostname_override='"$myip"'"/
-            ' /etc/kubernetes/kubelet
-
-            sed -i '
-              /^KUBE_MASTER=/ s/=.*/="--master=$KUBE_MASTER_IP:8080"/
-            ' /etc/kubernetes/apiserver
-
-            sed -i '
-              /^FLANNEL_ETCD=/ s|=.*|="http://$KUBE_MASTER_IP:4001"|
-            ' /etc/sysconfig/flanneld
-
-            cat >> /etc/environment <<EOF
-            KUBERNETES_MASTER=http://$KUBE_MASTER_IP:8080
-            EOF
-
-            # make centos user a member of the docker group
-            # (so you can run docker commands as the centos user)
-            if ! grep -q docker /etc/group; then
-              grep docker /lib/group >> /etc/group
-            fi
-            usermod -G docker centos
-
-            cat > /usr/local/bin/flanneld-waiter <<'EOF'
-            #!/bin/sh
-            
-            while ! [ -f /run/flannel/subnet.env ]; do
-              echo "waiting for flanneld"
-              sleep 1
-            done
-
-            echo flanneld is active
-
-            exit 0
-            EOF
-
-            chmod 755 /usr/local/bin/flanneld-waiter
-
-            cat > /etc/systemd/system/flanneld-waiter.service <<'EOF'
-            [Unit]
-            Description=Wait for flanneld to provide subnet/mtu information
-            After=network.target flanneld.service
-            Requires=flanneld.service
-
-            [Service]
-            Type=oneshot
-            ExecStart=/usr/local/bin/flanneld-waiter
-
-            [Install]
-            WantedBy=multi-user.target
-            EOF
-
-            cat > /etc/systemd/system/docker.service <<'EOF'
-            [Unit]
-            Description=Docker Application Container Engine
-            Documentation=http://docs.docker.com
-            After=network.target docker.socket flanneld-waiter.service
-            Requires=docker.socket flanneld-waiter.service
-
-            [Service]
-            Type=notify
-            EnvironmentFile=-/etc/sysconfig/docker
-            EnvironmentFile=-/etc/sysconfig/docker-storage
-            EnvironmentFile=-/run/flannel/subnet.env
-            ExecStart=/usr/bin/docker -d -H fd:// --bip $FLANNEL_SUBNET --mtu $FLANNEL_MTU $OPTIONS $DOCKER_STORAGE_OPTIONS
-            Restart=on-failure
-            LimitNOFILE=1048576
-            LimitNPROC=1048576
-
-            [Install]
-            WantedBy=multi-user.target
-            EOF
-
-            echo reloading systemd
-            systemctl daemon-reload
-
-            # docker is already enabled and possibly running on centos atomic host
-            # so we need to stop it first and delete the docker0 bridge (which will
-            # be re-created using the flannel-provided subnet).
-            echo stopping docker
-            systemctl stop docker
-            ip link del docker0
-
-            echo starting services
-            for service in flanneld-waiter flanneld docker.socket kubelet kube-proxy; do
-              systemctl enable $service
-              systemctl --no-block start $service
-            done
-
-            echo notifying heat
-            curl -sf -X PUT -H 'Content-Type: application/json' \
-              --data-binary '{"Status": "SUCCESS",
-                "Reason": "Setup complete",
-                "Data": "OK", "UniqueId": "00000"}' \
-              "$WAIT_HANDLE"
-          params:
-            "$ALLOW_PRIV": {get_param: allow_priv}
-            "$KUBE_MASTER_IP": {get_param: kube_master_ip}
-            "$WAIT_HANDLE": {get_resource: node_wait_handle}
       networks:
         - port:
             get_resource: kube_node_eth0