diff --git a/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml b/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml index 70dfdb1..9d8d45f 100644 --- a/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml +++ b/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml @@ -12,22 +12,4 @@ spec: kind: OpenStackMachineTemplate apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 name: {{ include "openstack-cluster.controlplane.mt.name" . }} - kubeadmConfigSpec: - initConfiguration: - nodeRegistration: - name: '{{ "{{" }} local_hostname {{ "}}" }}' - kubeletExtraArgs: - cloud-provider: external - clusterConfiguration: - imageRepository: {{ .Values.kubeImageRepository }} - apiServer: - extraArgs: - cloud-provider: external - controllerManager: - extraArgs: - cloud-provider: external - joinConfiguration: - nodeRegistration: - name: '{{ "{{" }} local_hostname {{ "}}" }}' - kubeletExtraArgs: - cloud-provider: external + kubeadmConfigSpec: {{ toYaml .Values.controlPlane.kubeadmConfigSpec | nindent 4 }} diff --git a/charts/openstack-cluster/templates/node-group/kubeadm-config-template.yaml b/charts/openstack-cluster/templates/node-group/kubeadm-config-template.yaml index 2f9a94f..0f78da5 100644 --- a/charts/openstack-cluster/templates/node-group/kubeadm-config-template.yaml +++ b/charts/openstack-cluster/templates/node-group/kubeadm-config-template.yaml @@ -1,16 +1,43 @@ +{{/* + In order for changes to the kubeadm config to be applied to existing machines, + kubeadm config templates must be treated as immutable. This means that we must + create a new one when the spec changes and update the ref in the corresponding + machine deployment, which will trigger a rollout of new machines with the new + kubeadm config. + + If the kubeadm config is updated in place, it will apply to new machine (e.g. on + a scale out operation), but will not be applied to existing machines. + + To do this, we create a new template whenever the checksum of the spec changes. +*/}} +{{- define "openstack-cluster.nodegroup.kct.spec" -}} +{{- $ctx := index . 0 -}} +{{- $nodeGroup := index . 1 -}} +template: + spec: {{ deepCopy $ctx.Values.nodeGroupDefaults.kubeadmConfigSpec | mustMerge (default dict $nodeGroup.kubeadmConfigSpec) | toYaml | nindent 4 }} +{{- end }} + +{{- define "openstack-cluster.nodegroup.kct.checksum" -}} +{{- include "openstack-cluster.nodegroup.kct.spec" . | sha256sum }} +{{- end }} + +{{- define "openstack-cluster.nodegroup.kct.name" -}} +{{- $ctx := index . 0 }} +{{- $nodeGroup := index . 1 }} +{{- $checksum := include "openstack-cluster.nodegroup.kct.checksum" . }} +{{- include "openstack-cluster.componentName" (list $ctx $nodeGroup.name) }}-{{ trunc 8 $checksum }} +{{- end }} + {{- range $nodeGroup := .Values.nodeGroups }} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 kind: KubeadmConfigTemplate metadata: - name: {{ include "openstack-cluster.componentName" (list $ $nodeGroup.name) }} + name: {{ include "openstack-cluster.nodegroup.kct.name" (list $ $nodeGroup) }} labels: {{ include "openstack-cluster.nodeGroupLabels" (list $ $nodeGroup.name) | nindent 4 }} -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - name: '{{ "{{" }} local_hostname {{ "}}" }}' - kubeletExtraArgs: - cloud-provider: external + annotations: + capi.stackhpc.com/template-checksum: {{ include "openstack-cluster.nodegroup.kct.checksum" (list $ $nodeGroup) }} + # Ensure that the previous templates are kept around + helm.sh/resource-policy: keep +spec: {{ include "openstack-cluster.nodegroup.kct.spec" (list $ $nodeGroup) | nindent 2 }} {{- end }} diff --git a/charts/openstack-cluster/templates/node-group/machine-deployment.yaml b/charts/openstack-cluster/templates/node-group/machine-deployment.yaml index e95b228..efdfa54 100644 --- a/charts/openstack-cluster/templates/node-group/machine-deployment.yaml +++ b/charts/openstack-cluster/templates/node-group/machine-deployment.yaml @@ -21,7 +21,7 @@ spec: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 kind: KubeadmConfigTemplate - name: {{ include "openstack-cluster.componentName" (list $ $nodeGroup.name) }} + name: {{ include "openstack-cluster.nodegroup.kct.name" (list $ $nodeGroup) }} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 kind: OpenStackMachineTemplate diff --git a/charts/openstack-cluster/templates/node-group/openstack-machine-template.yaml b/charts/openstack-cluster/templates/node-group/openstack-machine-template.yaml index 7ed28c0..cf3e1d6 100644 --- a/charts/openstack-cluster/templates/node-group/openstack-machine-template.yaml +++ b/charts/openstack-cluster/templates/node-group/openstack-machine-template.yaml @@ -3,8 +3,8 @@ To do this, we create a new template whenever the checksum of the spec changes. */}} {{- define "openstack-cluster.nodegroup.mt.spec" -}} -{{- $ctx := index . 0 }} -{{- $nodeGroup := index . 1 }} +{{- $ctx := index . 0 -}} +{{- $nodeGroup := index . 1 -}} template: spec: identityRef: @@ -38,6 +38,5 @@ metadata: capi.stackhpc.com/template-checksum: {{ include "openstack-cluster.nodegroup.mt.checksum" (list $ $nodeGroup) }} # Ensure that the previous machine templates are kept around helm.sh/resource-policy: keep -spec: - {{- include "openstack-cluster.nodegroup.mt.spec" (list $ $nodeGroup) | nindent 2 }} +spec: {{ include "openstack-cluster.nodegroup.mt.spec" (list $ $nodeGroup) | nindent 2 }} {{- end }} diff --git a/charts/openstack-cluster/values.yaml b/charts/openstack-cluster/values.yaml index 808900a..40581ad 100644 --- a/charts/openstack-cluster/values.yaml +++ b/charts/openstack-cluster/values.yaml @@ -16,9 +16,6 @@ machineSSHKeyName: # This should match the version of kubelet in the image kubernetesVersion: -# The repository to pull Kubernetes system container images from -kubeImageRepository: k8s.gcr.io - # Values for the Kubernetes cluster network kubeNetwork: # By default, use the private network range 172.16.0.0/12 for the cluster network @@ -74,14 +71,43 @@ controlPlane: machineCount: 3 # The flavor to use for control plane machines machineFlavor: + # The kubeadm config specification for the control plane + # By default, this uses a simple configuration that just enables the external cloud provider + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + name: '{{ local_hostname }}' + kubeletExtraArgs: + cloud-provider: external + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external + joinConfiguration: + nodeRegistration: + name: '{{ local_hostname }}' + kubeletExtraArgs: + cloud-provider: external # Defaults for node groups -# Each of these can be overridden in the node group specification +# Each of these can be overridden in the specification for an individual node group nodeGroupDefaults: # The failure domain for the node group failureDomain: nova # The flavor to use for machines in the node group machineFlavor: + # The default kubeadm config specification for worker nodes + # This will be merged with any configuration given for specific node groups + # By default, this uses a simple configuration that just enables the external cloud provider + kubeadmConfigSpec: + joinConfiguration: + nodeRegistration: + name: '{{ local_hostname }}' + kubeletExtraArgs: + cloud-provider: external # The worker node groups for the cluster nodeGroups: