Allow kubeadm configuration to be customised

charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml

@@ -12,22 +12,4 @@ spec:
       kind: OpenStackMachineTemplate
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
       name: {{ include "openstack-cluster.controlplane.mt.name" . }}
-  kubeadmConfigSpec:
-    initConfiguration:
-      nodeRegistration:
-        name: '{{ "{{" }} local_hostname {{ "}}" }}'
-        kubeletExtraArgs:
-          cloud-provider: external
-    clusterConfiguration:
-      imageRepository: {{ .Values.kubeImageRepository }}
-      apiServer:
-        extraArgs:
-          cloud-provider: external
-      controllerManager:
-        extraArgs:
-          cloud-provider: external
-    joinConfiguration:
-      nodeRegistration:
-        name: '{{ "{{" }} local_hostname {{ "}}" }}'
-        kubeletExtraArgs:
-          cloud-provider: external
+  kubeadmConfigSpec: {{ toYaml .Values.controlPlane.kubeadmConfigSpec | nindent 4 }}

charts/openstack-cluster/templates/node-group/kubeadm-config-template.yaml

@@ -1,16 +1,43 @@
+{{/*
+  In order for changes to the kubeadm config to be applied to existing machines,
+  kubeadm config templates must be treated as immutable. This means that we must
+  create a new one when the spec changes and update the ref in the corresponding
+  machine deployment, which will trigger a rollout of new machines with the new
+  kubeadm config.
+
+  If the kubeadm config is updated in place, it will apply to new machine (e.g. on
+  a scale out operation), but will not be applied to existing machines.
+
+  To do this, we create a new template whenever the checksum of the spec changes.
+*/}}
+{{- define "openstack-cluster.nodegroup.kct.spec" -}}
+{{- $ctx := index . 0 -}}
+{{- $nodeGroup := index . 1 -}}
+template:
+  spec: {{ deepCopy $ctx.Values.nodeGroupDefaults.kubeadmConfigSpec | mustMerge (default dict $nodeGroup.kubeadmConfigSpec) | toYaml | nindent 4 }}
+{{- end }}
+
+{{- define "openstack-cluster.nodegroup.kct.checksum" -}}
+{{- include "openstack-cluster.nodegroup.kct.spec" . | sha256sum }}
+{{- end }}
+
+{{- define "openstack-cluster.nodegroup.kct.name" -}}
+{{- $ctx := index . 0 }}
+{{- $nodeGroup := index . 1 }}
+{{- $checksum := include "openstack-cluster.nodegroup.kct.checksum" . }}
+{{- include "openstack-cluster.componentName" (list $ctx $nodeGroup.name) }}-{{ trunc 8 $checksum }}
+{{- end }}
+
 {{- range $nodeGroup := .Values.nodeGroups }}
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
 kind: KubeadmConfigTemplate
 metadata:
-  name: {{ include "openstack-cluster.componentName" (list $ $nodeGroup.name) }}
+  name: {{ include "openstack-cluster.nodegroup.kct.name" (list $ $nodeGroup) }}
   labels: {{ include "openstack-cluster.nodeGroupLabels" (list $ $nodeGroup.name) | nindent 4 }}
-spec:
-  template:
-    spec:
-      joinConfiguration:
-        nodeRegistration:
-          name: '{{ "{{" }} local_hostname {{ "}}" }}'
-          kubeletExtraArgs:
-            cloud-provider: external
+  annotations:
+    capi.stackhpc.com/template-checksum: {{ include "openstack-cluster.nodegroup.kct.checksum" (list $ $nodeGroup) }}
+    # Ensure that the previous templates are kept around
+    helm.sh/resource-policy: keep
+spec: {{ include "openstack-cluster.nodegroup.kct.spec" (list $ $nodeGroup) | nindent 2 }}
 {{- end }}

charts/openstack-cluster/templates/node-group/machine-deployment.yaml

@@ -21,7 +21,7 @@ spec:
         configRef:
           apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
           kind: KubeadmConfigTemplate
-          name: {{ include "openstack-cluster.componentName" (list $ $nodeGroup.name) }}
+          name: {{ include "openstack-cluster.nodegroup.kct.name" (list $ $nodeGroup) }}
       infrastructureRef:
         apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
         kind: OpenStackMachineTemplate

charts/openstack-cluster/templates/node-group/openstack-machine-template.yaml

@@ -3,8 +3,8 @@
   To do this, we create a new template whenever the checksum of the spec changes.
 */}}
 {{- define "openstack-cluster.nodegroup.mt.spec" -}}
-{{- $ctx := index . 0 }}
-{{- $nodeGroup := index . 1 }}
+{{- $ctx := index . 0 -}}
+{{- $nodeGroup := index . 1 -}}
 template:
   spec:
     identityRef:
@@ -38,6 +38,5 @@ metadata:
     capi.stackhpc.com/template-checksum: {{ include "openstack-cluster.nodegroup.mt.checksum" (list $ $nodeGroup) }}
     # Ensure that the previous machine templates are kept around
     helm.sh/resource-policy: keep
-spec:
-  {{- include "openstack-cluster.nodegroup.mt.spec" (list $ $nodeGroup) | nindent 2 }}
+spec: {{ include "openstack-cluster.nodegroup.mt.spec" (list $ $nodeGroup) | nindent 2 }}
 {{- end }}

charts/openstack-cluster/values.yaml

@@ -16,9 +16,6 @@ machineSSHKeyName:
 # This should match the version of kubelet in the image
 kubernetesVersion:
 
-# The repository to pull Kubernetes system container images from 
-kubeImageRepository: k8s.gcr.io
-
 # Values for the Kubernetes cluster network
 kubeNetwork:
   # By default, use the private network range 172.16.0.0/12 for the cluster network
@@ -74,14 +71,43 @@ controlPlane:
   machineCount: 3
   # The flavor to use for control plane machines
   machineFlavor:
+  # The kubeadm config specification for the control plane
+  # By default, this uses a simple configuration that just enables the external cloud provider
+  kubeadmConfigSpec:
+    initConfiguration:
+      nodeRegistration:
+        name: '{{ local_hostname }}'
+        kubeletExtraArgs:
+          cloud-provider: external
+    clusterConfiguration:
+      apiServer:
+        extraArgs:
+          cloud-provider: external
+      controllerManager:
+        extraArgs:
+          cloud-provider: external
+    joinConfiguration:
+      nodeRegistration:
+        name: '{{ local_hostname }}'
+        kubeletExtraArgs:
+          cloud-provider: external
 
 # Defaults for node groups
-# Each of these can be overridden in the node group specification
+# Each of these can be overridden in the specification for an individual node group
 nodeGroupDefaults:
   # The failure domain for the node group
   failureDomain: nova
   # The flavor to use for machines in the node group
   machineFlavor:
+  # The default kubeadm config specification for worker nodes
+  # This will be merged with any configuration given for specific node groups
+  # By default, this uses a simple configuration that just enables the external cloud provider
+  kubeadmConfigSpec:
+    joinConfiguration:
+      nodeRegistration:
+        name: '{{ local_hostname }}'
+        kubeletExtraArgs:
+          cloud-provider: external
 
 # The worker node groups for the cluster
 nodeGroups: