From a0f3ca46240f924c2e05956132e89510ec7ffc8e Mon Sep 17 00:00:00 2001
From: Matt Pryor <matt@stackhpc.com>
Date: Tue, 4 Jun 2024 16:57:31 +0100
Subject: [PATCH] Use kube-proxy replacement by default with Cilium (#343)

---
 .../cluster-addons/templates/cni/cilium.yaml  | 17 ++++++++++++++
 .../openstack-cluster/templates/_helpers.tpl  |  1 -
 .../control-plane/kubeadm-control-plane.yaml  | 23 +++++++++++++++++--
 3 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/charts/cluster-addons/templates/cni/cilium.yaml b/charts/cluster-addons/templates/cni/cilium.yaml
index ead045a..eb611a1 100644
--- a/charts/cluster-addons/templates/cni/cilium.yaml
+++ b/charts/cluster-addons/templates/cni/cilium.yaml
@@ -1,4 +1,14 @@
 {{- if and .Values.cni.enabled (eq .Values.cni.type "cilium") }}
+{{/*
+  Use the kube-proxy replacement by default
+
+  NOTE(mkjpryor)
+  When using the kube-proxy replacement Cilium must be told the host and port for the
+  Kubernetes API as it is responsible for handling service traffic
+  We can use the templating support in the addon provider to extract this dynamically
+  from the CAPI cluster resource spec
+*/}}
+{{- $kubeProxyReplacement := dig "kubeProxyReplacement" true .Values.cni.cilium.release.values }}
 ---
 apiVersion: v1
 kind: Secret
@@ -9,8 +19,15 @@ metadata:
     addons.stackhpc.com/watch: ""
 stringData:
   defaults: |
+    # Use IPs allocated to each node by Kubernetes
+    # This respects the pod CIDR specified in the CAPI cluster resource
     ipam:
       mode: kubernetes
+    {{- if $kubeProxyReplacement }}
+    kubeProxyReplacement: true
+    k8sServiceHost: {{ "{{" }} cluster.spec.controlPlaneEndpoint.host {{ "}}" }}
+    k8sServicePort: {{ "{{" }} cluster.spec.controlPlaneEndpoint.port {{ "}}" }}
+    {{- end }}
   overrides: |
     {{- toYaml .Values.cni.cilium.release.values | nindent 4 }}
 ---
diff --git a/charts/openstack-cluster/templates/_helpers.tpl b/charts/openstack-cluster/templates/_helpers.tpl
index d460df8..4585c60 100644
--- a/charts/openstack-cluster/templates/_helpers.tpl
+++ b/charts/openstack-cluster/templates/_helpers.tpl
@@ -346,7 +346,6 @@ webhooks and policies for audit logging can be added here.
   clusterConfiguration:
     apiServer:
       extraArgs:
-        cloud-provider: external
 {{- if $authWebhook }}
         authorization-mode: Node,Webhook,RBAC
 {{- if eq $authWebhook "k8s-keystone-auth" }}
diff --git a/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml b/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml
index 87eb661..aaf4c59 100644
--- a/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml
+++ b/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml
@@ -8,8 +8,8 @@ joinConfiguration: {{ include "openstack-cluster.nodeRegistration.nodeLabels" .
 {{- define "openstack-cluster.controlplane.kubeadmConfigSpec.kubeProxyConfiguration" -}}
 {{- $ctx := index . 0 }}
 {{- $osDistro := index . 1 }}
-{{- $kubeProxyConfigurationPath := ternary "/etc/kube-proxy-configuration.yaml" "/run/kubeadm/kube-proxy-configuration.yaml" (eq $osDistro "flatcar") }}
-{{- $kubeadmPath := ternary "/etc/kubeadm.yml" "/run/kubeadm/kubeadm.yaml" (eq $osDistro "flatcar") }}
+{{- $kubeProxyConfigurationPath := eq $osDistro "flatcar" | ternary "/etc/kube-proxy-configuration.yaml" "/run/kubeadm/kube-proxy-configuration.yaml" }}
+{{- $kubeadmPath := eq $osDistro "flatcar" | ternary "/etc/kubeadm.yml" "/run/kubeadm/kubeadm.yaml" }}
 {{- with $ctx.kubeProxyConfiguration }}
 files:
   - path: {{ $kubeProxyConfigurationPath }}
@@ -52,6 +52,24 @@ mounts:
 {{- end }}
 {{- end }}
 
+{{/*
+  NOTE(mkjpryor)
+  When Cilium is enabled with the kube-proxy replacement, we need to skip kube-proxy
+*/}}
+{{- define "openstack-cluster.controlplane.kubeadmConfigSpec.cilium" -}}
+{{-
+  if and
+    .Values.addons.enabled
+    (dig "cni" "enabled" true .Values.addons)
+    (dig "cni" "type" "calico" .Values.addons | eq "cilium")
+    (dig "cni" "cilium" "release" "values" "kubeProxyReplacement" true .Values.addons)
+}}
+initConfiguration:
+  skipPhases:
+    - addon/kube-proxy
+{{- end }}
+{{- end }}
+
 ---
 apiVersion: controlplane.cluster.x-k8s.io/v1beta1
 kind: KubeadmControlPlane
@@ -85,6 +103,7 @@ spec:
         list
           (include "openstack-cluster.controlplane.kubeadmConfigSpec.nodeLabels" . | fromYaml)
           (include "openstack-cluster.controlplane.kubeadmConfigSpec.etcd" . | fromYaml)
+          (include "openstack-cluster.controlplane.kubeadmConfigSpec.cilium" . | fromYaml)
           (include "openstack-cluster.kubeadmConfigSpec" (list . .Values.controlPlane.kubeadmConfigSpec) | fromYaml)
           (include "openstack-cluster.osDistroKubeadmConfigSpec" (list . ) | fromYaml)
           (include "openstack-cluster.patchConfigSpec"  (list .) | fromYaml)