Update tolerations and priority classes

* Give kube-proxy a blanket toleration
* Replace scheduler.alpha.kubernetes.io/critical-pod annotation with
    priorityClassName: system-node-critical

Change-Id: I810333913c09531eefa1ded014fe090d4cca7f7d

charts/apiserver/templates/daemonset.yaml

@@ -42,7 +42,6 @@ spec:
 {{ $labels | indent 8 }}
       annotations:
         {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
 {{ dict "envAll" $envAll "podName" "kubernetes_apiserver_anchor" "containerNames" (list "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
@@ -52,6 +51,7 @@ spec:
         {{ .Values.labels.kubernetes_apiserver.node_selector_key }}: {{ .Values.labels.kubernetes_apiserver.node_selector_value }}
       dnsPolicy: {{ .Values.anchor.dns_policy }}
       hostNetwork: true
+      priorityClassName: system-node-critical
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule

charts/controller_manager/templates/daemonset.yaml

@@ -39,7 +39,6 @@ spec:
 {{ $labels | indent 8 }}
       annotations:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
 {{ dict "envAll" $envAll "podName" "kubernetes-controller-manager-anchor" "containerNames" (list "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
@@ -49,6 +48,7 @@ spec:
         {{ .Values.labels.controller_manager.node_selector_key }}: {{ .Values.labels.controller_manager.node_selector_value }}
       dnsPolicy: {{ .Values.anchor.dns_policy }}
       hostNetwork: true
+      priorityClassName: system-node-critical
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule

charts/etcd/templates/daemonset-anchor.yaml

@@ -41,7 +41,6 @@ spec:
       annotations:
 {{ dict "envAll" $envAll "podName" "etcd-anchor" "containerNames" (list "etcdctl") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
-        scheduler.alpha.kubernetes.io/critical-pod: ''
 {{- if .Values.manifests.configmap_bin }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
 {{- end }}
@@ -59,6 +58,7 @@ spec:
       {{- end }}
       nodeSelector:
         {{ .Values.labels.anchor.node_selector_key }}: {{ .Values.labels.anchor.node_selector_value }}
+      priorityClassName: system-node-critical
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule

charts/etcd/templates/tests/test-etcd-health.yaml

@@ -25,7 +25,6 @@ metadata:
   name: "{{ .Release.Name }}-etcd-test"
   annotations:
     "helm.sh/hook": "test"
-    scheduler.alpha.kubernetes.io/critical-pod: ''
     {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
 {{ dict "envAll" $envAll "podName" "etcd-test" "containerNames" (list "etcd-test") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }}
   labels:

charts/haproxy/templates/daemonset.yaml

@@ -36,10 +36,10 @@ spec:
 {{ $labels | indent 8 }}
       annotations:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
-        scheduler.alpha.kubernetes.io/critical-pod: ''
 {{ dict "envAll" $envAll "podName" "haproxy-anchor" "containerNames" (list "haproxy-perms" "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
     spec:
 {{ dict "envAll" $envAll "application" "haproxy_anchor" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      priorityClassName: system-node-critical
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule

charts/proxy/templates/daemonset.yaml

@@ -45,7 +45,6 @@ spec:
       annotations:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
 {{ dict "envAll" $envAll "podName" "kubernetes-proxy" "containerNames" (list "proxy") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
-        scheduler.alpha.kubernetes.io/critical-pod: ''
 {{- if .Values.manifests.configmap_proxy }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
 {{- end }}
@@ -54,14 +53,9 @@ spec:
       hostNetwork: true
       shareProcessNamespace: true
       dnsPolicy: Default
+      priorityClassName: system-node-critical
       tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-        - key: CriticalAddonsOnly
-          operator: Exists
-        - key: node.kubernetes.io/not-ready
-          operator: Exists
-          effect: NoSchedule
+        - operator: Exists
       containers:
       - name: proxy
         image: {{ .Values.images.tags.proxy }}

charts/scheduler/templates/sched-anchor.yaml

@@ -34,7 +34,6 @@ spec:
     metadata:
       annotations:
 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
 {{ dict "envAll" $envAll "podName" "scheduler" "containerNames" (list "anchor") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
@@ -46,6 +45,7 @@ spec:
       dnsPolicy: {{ .Values.anchor.dns_policy }}
       nodeSelector:
         {{ .Values.labels.scheduler.node_selector_key }}: {{ .Values.labels.scheduler.node_selector_value }}
+      priorityClassName: system-node-critical
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule

promenade/templates/roles/common/etc/kubernetes/manifests/haproxy.yaml

@@ -4,10 +4,9 @@ kind: Pod
 metadata:
   name: haproxy
   namespace: kube-system
-  annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
 spec:
   hostNetwork: true
+  priorityClassName: system-node-critical
   containers:
     - name: haproxy
       image: {{ config['HostSystem:images.haproxy'] }}

promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml

@@ -8,10 +8,9 @@ metadata:
     application: kubernetes
     component: apiserver
     kubernetes-apiserver-service: enabled
-  annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
 spec:
   hostNetwork: true
+  priorityClassName: system-node-critical
   containers:
     - name: kube-apiserver
       image: {{ config['Genesis:images.kubernetes.apiserver'] }}

promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-controller-manager.yaml

@@ -8,10 +8,9 @@ metadata:
     tier: control-plane
     application: kubernetes
     component: kube-controller-manager
-  annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
 spec:
   hostNetwork: true
+  priorityClassName: system-node-critical
   containers:
     - name: kube-controller-manager
       image: {{ config['Genesis:images.kubernetes.controller-manager'] }}

promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-scheduler.yaml

@@ -8,10 +8,9 @@ metadata:
     tier: control-plane
     application: kubernetes
     component: kube-scheduler
-  annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
 spec:
   hostNetwork: true
+  priorityClassName: system-node-critical
   containers:
     - name: kube-scheduler
       image: {{ config['Genesis:images.kubernetes.scheduler'] }}