Added cockroachdb cluster setup across multiple clouds

This patch does the following:
1. Added the apps role so that k8s app deployment can be placed in the role
2. Added cockroachdb-init-pod deployment inthe post role since this has to
   be deployed first.
3. Added cockroachdb-pod deployment in the apps role
4. Allow this workload to stand up a standalone cockroachdb cluster or
   join in an existing cockroachdb cluster
5. Added the cockroachdb load generator container so that once the cluster
   is started, there will be some load generated.
6. Added a way to pre-allocating floating IP addresses so that the workload
   can use pre-allocated floating IPs for VMs.

Change-Id: Ifa9eeb9d761d9801cab580445e6c43c8cf1dfdaa

workloads/ansible/shade/k8s/roles/apps/tasks/main.yml

@@ -0,0 +1,42 @@
+---
+- name: Setup few variables for coreos target
+  set_fact:
+    master_ip: "{{ groups['cmasters'][0] }}"
+    public_ip: "{{ ansible_host }}"
+  when: app_env.target_os == "coreos"
+
+- name: Setup few variables for ubuntu target
+  set_fact:
+    master_ip: "{{ groups['umasters'][0] }}"
+    public_ip: "{{ ansible_host }}"
+  when: app_env.target_os == "ubuntu"
+
+- name: Setup private IP variable for master node
+  set_fact:
+    private_ip: "{{ hostvars[master_ip].inter_ip }}"
+
+- name: Setup first node IP when stand alone
+  set_fact:
+    first_node_ip: "{{ master_ip }}"
+  when: app_env.app_setting.own_cluster == True
+
+- name: Setup first node IP when join others
+  set_fact:
+    first_node_ip: "{{ app_env.app_setting.public_node }}"
+  when: app_env.app_setting.own_cluster == False
+
+- name: Upload app configuration files
+  template:
+    src: "roles/apps/templates/{{ item }}.j2"
+    dest: "/etc/kubernetes/{{ item }}.yaml"
+    mode: 0644
+  with_items:
+    - cockroachdb-pod
+
+- name: Create the app pod
+  command: >-
+    /opt/bin/kubectl --server="{{ private_ip }}:8080" create
+    -f "/etc/kubernetes/{{ item }}.yaml"
+  with_items:
+    - cockroachdb-pod
+

workloads/ansible/shade/k8s/roles/apps/templates/cockroachdb-pod.j2

@@ -0,0 +1,45 @@
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cockroachdb-{{ inter_name }}
+  labels:
+    app: cockroachdb
+spec:
+  nodeName: {{ inter_name }}
+  restartPolicy: Always
+  containers:
+  - name: cockroachdb
+    image: cockroachdb/cockroach
+    imagePullPolicy: IfNotPresent
+    ports:
+    - containerPort: 26257
+      hostPort: 26257
+      name: grpc
+    - containerPort: 8080
+      hostPort: 8090
+      name: http
+    volumeMounts:
+    - name: datadir
+      mountPath: /cockroach/cockroach-data
+    command:
+      - "/bin/bash"
+      - "-ecx"
+      - |
+        CRARGS=("start" "--logtostderr" "--insecure" "--http-host" "0.0.0.0" )
+        CRARGS+=("--advertise-host" "{{ public_ip }}" )
+        CRARGS+=("--locality" "cloud={{ env }}")
+        CRARGS+=("--join" "{{ first_node_ip }}")
+        exec /cockroach/cockroach ${CRARGS[*]}
+  - name: loadgen
+    image: cockroachdb/interop-demo:1.1
+    imagePullPolicy: IfNotPresent
+    command:
+    - /kv
+    - --read-percent=50
+    - --max-rate=500
+  terminationGracePeriodSeconds: 30
+  volumes:
+  - name: datadir
+    hostPath:
+      path: /storage/cockroachdb

workloads/ansible/shade/k8s/roles/post/tasks/apply.yml

@@ -12,7 +12,7 @@
   with_items:
     - dnscontroller
     - dashboard
-    - cockroachdb
+    - cockroachdb-init-pod
 
 - name: Label the master node
   command: >-
@@ -26,5 +26,9 @@
   with_items:
     - dnscontroller
     - dashboard
-    - cockroachdb
 
+- name: Setup first cockroachdb node
+  command: >-
+    /opt/bin/kubectl --server={{ private_ip }}:8080 create
+    -f /etc/kubernetes/cockroachdb-init-pod.yaml
+  when: app_env.app_setting.own_cluster == True

workloads/ansible/shade/k8s/roles/post/templates/cockroachdb-init-pod.j2

@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cockroachdb-{{ inter_name }}
+  labels:
+    app: cockroachdb
+spec:
+  nodeName: {{ inter_name }}
+  restartPolicy: Always
+  containers:
+  - name: cockroachdb
+    image: cockroachdb/cockroach
+    imagePullPolicy: IfNotPresent
+    ports:
+    - containerPort: 26257
+      hostPort: 26257
+      name: grpc
+    - containerPort: 8080
+      hostPort: 8090
+      name: http
+    volumeMounts:
+    - name: datadir
+      mountPath: /cockroach/cockroach-data
+    command:
+      - "/bin/bash"
+      - "-ecx"
+      - |
+        CRARGS=("start" "--logtostderr" "--insecure" "--http-host")
+        CRARGS+=("0.0.0.0" "--advertise-host" "{{ public_ip }}")
+        CRARGS+=("--locality" "cloud={{ env }}")
+        exec /cockroach/cockroach ${CRARGS[*]}
+  - name: loadgen
+    image: cockroachdb/interop-demo:1.1
+    imagePullPolicy: IfNotPresent
+    command:
+    - /kv
+    - --splits=100
+    - --read-percent=50
+    - --max-rate=500
+  terminationGracePeriodSeconds: 30
+  volumes:
+  - name: datadir
+    hostPath:
+      path: /storage/cockroachdb

workloads/ansible/shade/k8s/roles/prepare/tasks/apply.yml

@@ -63,11 +63,12 @@
     - { p_min: 53, p_max: 53, dir: ingress, protocol: udp }
     - { p_min: 53, p_max: 53, dir: egress, protocol: udp }
     - { p_min: 8080, p_max: 8080, dir: ingress, protocol: tcp }
+    - { p_min: 8090, p_max: 8090, dir: ingress, protocol: tcp }
     - { p_min: 8285, p_max: 8285, dir: ingress, protocol: udp }
     - { p_min: 2379, p_max: 2380, dir: ingress, protocol: tcp }
     - { p_min: 2379, p_max: 2380, dir: egress, protocol: tcp }
     - { p_min: 10250, p_max: 10250, dir: ingress, protocol: tcp }
-    - { p_min: 30000, p_max: 32767, dir: ingress, protocol: tcp }
+    - { p_min: 20000, p_max: 32767, dir: ingress, protocol: tcp }
     - { p_min: -1, p_max: -1, dir: ingress, protocol: icmp }
     - { p_min: -1, p_max: -1, dir: egress, protocol: icmp }
 

workloads/ansible/shade/k8s/roles/provision/tasks/apply.yml

@@ -25,6 +25,7 @@
     flavor: "{{ app_env.flavor_name }}"
     network: "{{ app_env.private_net_name }}"
     floating_ip_pools: "{{ app_env.public_net_name | default(omit) }}"
+    floating_ips: "{{ app_env.app_setting.ips[inventory_hostname] }}"
     reuse_ips: False
     userdata: "{{ lookup('template', tp_path) }}"
     config_drive: "{{ app_env.config_drive | default('no') }}"

workloads/ansible/shade/k8s/roles/provision/tasks/destroy.yml

@@ -1,5 +1,9 @@
 ---
 
+- name: Setup release floating IP flag
+  set_fact:
+    ip_flag: "{{ app_env.app_setting.ips[inventory_hostname] == '' }}"
+
 - name: Destroy the OpenStack VM
   os_server:
     state: "absent"
@@ -9,7 +13,7 @@
     validate_certs: "{{ app_env.validate_certs }}"
     name: "{{ inventory_hostname }}"
     image: "{{ app_env.image_name }}"
-    delete_fip: True
+    delete_fip: "{{ ip_flag }}"
     key_name: "k8s"
     timeout: 200
     network: "{{ app_env.private_net_name }}"

workloads/ansible/shade/k8s/site.yml

@@ -110,6 +110,19 @@
   environment: "{{ proxy_env }}"
   tags: "post"
 
+- name: Start up applications
+  hosts: cworkers, uworkers
+  gather_facts: true
+  user: "{{ app_env.ssh_user }}"
+  become: true
+  become_user: root
+  vars_files:
+    - "vars/{{ env }}.yml"
+  roles:
+    - apps
+  environment: "{{ proxy_env }}"
+  tags: "apps"
+
 - name: Inform the installer
   hosts: cloud
   connection: local

workloads/ansible/shade/k8s/vars/coreos.yml

@@ -40,7 +40,25 @@ app_env: {
   dns_service_ip: "172.16.0.4",
   dashboard_service_ip: "172.16.0.5",
 
+  app_setting: {
+    public_node: "",
+    own_cluster: True,
+    # The following section shows how to pre allocate floating IPs for each
+    # server. If you wish not to pre allocate floating IPs or your cloud
+    # does not support floating IPs, leave them empty. The stack_size
+    # above should dictate how many worker nodes should be. For example, if
+    # your stack_size is 10, you will need to add worker-1 to worker-9.
+    ips: {
+      master: [],
+      worker-1: [],
+      worker-2: []
+    }
+  },
+
   # The following section shows an example when use a local repo.
+  # If you have exported some container images such as images that being used
+  # by this workload, you can place the url point to tar.gz file for
+  # cimages_repo
   cimages_repo: "http://10.0.10.12/cimages.tar.gz",
   flannel_repo: "http://10.0.10.12/flannel-v0.7.0-linux-amd64.tar.gz",
   k8s_repo: "http://10.0.10.12/v1.5.4/"

workloads/ansible/shade/k8s/vars/ubuntu.yml

@@ -40,7 +40,25 @@ app_env: {
   dns_service_ip: "172.16.0.4",
   dashboard_service_ip: "172.16.0.5",
 
+  app_setting: {
+    public_node: "",
+    own_cluster: True,
+    # The following section shows how to pre allocate floating IPs for each
+    # server. If you wish not to pre allocate floating IPs or your cloud
+    # does not support floating IPs, leave them empty. The stack_size
+    # above should dictate how many worker nodes should be. For example, if
+    # your stack_size is 10, you will need to add worker-1 to worker-9.
+    ips: {
+      master: [],
+      worker-1: [],
+      worker-2: []
+    }
+  },
+
   # The following section shows an example when use a remote repo.
+  # If you have exported some container images such as images that being used
+  # by this workload, you can place the url point to tar.gz file for
+  # cimages_repo
   cimages_repo: "",
   flannel_repo: "https://github.com/coreos/flannel/releases/download/v0.7.0/flannel-v0.7.0-linux-amd64.tar.gz",
   k8s_repo: "https://storage.googleapis.com/kubernetes-release/release/v1.5.3/bin/linux/amd64/"