Add Additional Liveness Probes for apiserver

- Updated apiserver-anchor with a liveness probe.
- Changed apiserver liveness probe to query kubectl.
   This allows the pod to restart if it looses access to etcd.

Change-Id: I0ef9cbc941a0533268e4f499a1333e88be3e43a3

charts/apiserver/templates/configmap-etc.yaml

@@ -25,4 +25,6 @@ metadata:
 data:
   kubernetes-apiserver.yaml: |+
 {{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  kubeconfig.yaml: |+
+{{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

charts/apiserver/templates/etc/_kubeconfig.yaml.tpl

@@ -0,0 +1,34 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: v1
+clusters:
+- cluster:
+    server: https://127.0.0.1:{{ .Values.network.kubernetes_apiserver.port }}
+    certificate-authority: pki/cluster-ca.pem
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: apiserver
+  name: apiserver@kubernetes
+current-context: apiserver@kubernetes
+kind: Config
+preferences: {}
+users:
+- name: apiserver
+  user:
+    client-certificate: pki/apiserver.pem
+    client-key: pki/apiserver-key.pem

charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl

@@ -33,6 +33,12 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
+        - name: NODENAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: KUBECONFIG
+          value: /etc/kubernetes/apiserver/kubeconfig.yaml
 
       command:
         {{- range .Values.command_prefix }}
@@ -81,13 +87,10 @@ spec:
           - /bin/bash
           - -c
           - |-
-            if [ ! -f /etc/kubernetes/apiserver/pki/apiserver-both.pem ]; then
-              cat /etc/kubernetes/apiserver/pki/apiserver-key.pem /etc/kubernetes/apiserver/pki/apiserver.pem > /etc/kubernetes/apiserver/pki/apiserver-both.pem
-            fi
-            echo -e 'GET /healthz HTTP/1.0\r\n' | socat - openssl:localhost:{{ .Values.network.kubernetes_apiserver.port }},cert=/etc/kubernetes/apiserver/pki/apiserver-both.pem,cafile=/etc/kubernetes/apiserver/pki/cluster-ca.pem | grep '200 OK'
+            kubectl get nodes ${NODENAME} | grep ${NODENAME}
             exit $?
-        failureThreshold: 2
-        initialDelaySeconds: 15
+        failureThreshold: 3
+        initialDelaySeconds: 60
         periodSeconds: 10
         successThreshold: 1
         timeoutSeconds: 10

charts/apiserver/templates/rbac.yaml

@@ -0,0 +1,42 @@
+{{/*
+Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: apiserver
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - get
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: apiserver
+  namespace: kube-system
+subjects:
+  - kind: User
+    name: apiserver
+    apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: apiserver
+  apiGroup: rbac.authorization.k8s.io

charts/apiserver/values.yaml

@@ -48,6 +48,8 @@ anchor:
       dest: /etc/kubernetes/apiserver/pki/etcd-client-key.pem
     - source: /tmp/etc/kubernetes-apiserver.yaml
       dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml
+    - source: /tmp/etc/kubeconfig.yaml
+      dest: /etc/kubernetes/apiserver/kubeconfig.yaml
 
 command_prefix:
   - /apiserver