From 443ff3e3e340c94c5cbb214d1e2a8b2a3937541d Mon Sep 17 00:00:00 2001 From: Stephen Taylor Date: Tue, 15 Aug 2023 12:55:26 -0600 Subject: [PATCH] [ceph] Use Helm toolkit functions for Ceph probes This change converts the readiness and liveness probes in the Ceph charts to use the functions from the Helm toolkit rather than having hard-coded probe definitions. This allows probe configs to be overridden in values.yaml without rebuilding charts. Change-Id: I68a01b518f12d33fe4f87f86494a5f4e19be982e --- ceph-client/Chart.yaml | 2 +- ceph-client/templates/deployment-mds.yaml | 21 +++++++++-------- ceph-client/values.yaml | 12 ++++++++++ ceph-mon/Chart.yaml | 2 +- ceph-mon/templates/daemonset-mon.yaml | 28 +++++++++++------------ ceph-mon/templates/deployment-mgr.yaml | 28 +++++++++++------------ ceph-mon/values.yaml | 26 +++++++++++++++++++++ ceph-osd/Chart.yaml | 2 +- ceph-osd/templates/daemonset-osd.yaml | 28 +++++++++++------------ ceph-osd/values.yaml | 15 ++++++++++++ releasenotes/notes/ceph-client.yaml | 1 + releasenotes/notes/ceph-mon.yaml | 1 + releasenotes/notes/ceph-osd.yaml | 1 + 13 files changed, 113 insertions(+), 54 deletions(-) diff --git a/ceph-client/Chart.yaml b/ceph-client/Chart.yaml index 4f48d2bc71..4e669d7329 100644 --- a/ceph-client/Chart.yaml +++ b/ceph-client/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Client name: ceph-client -version: 0.1.46 +version: 0.1.47 home: https://github.com/ceph/ceph-client ... diff --git a/ceph-client/templates/deployment-mds.yaml b/ceph-client/templates/deployment-mds.yaml index 2640c1c3d5..ba67a8d476 100644 --- a/ceph-client/templates/deployment-mds.yaml +++ b/ceph-client/templates/deployment-mds.yaml @@ -12,6 +12,16 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "livenessProbeTemplate" }} +tcpSocket: + port: 6800 +{{- end }} + +{{- define "readinessProbeTemplate" }} +tcpSocket: + port: 6800 +{{- end }} + {{- if and .Values.manifests.deployment_mds ( and .Values.deployment.ceph .Values.conf.features.mds) }} {{- $envAll := . }} @@ -100,15 +110,8 @@ spec: value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} ports: - containerPort: 6800 - livenessProbe: - tcpSocket: - port: 6800 - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - tcpSocket: - port: 6800 - timeoutSeconds: 5 +{{ dict "envAll" . "component" "ceph" "container" "ceph-mds" "type" "liveness" "probeTemplate" (include "livenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph" "container" "ceph-mds" "type" "readiness" "probeTemplate" (include "readinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} volumeMounts: - name: pod-tmp mountPath: /tmp diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 04d83bec83..0162ed2c93 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -179,6 +179,18 @@ pod: key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 60 + probes: + ceph: + ceph-mds: + readiness: + enabled: true + params: + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 60 + timeoutSeconds: 5 secrets: keyrings: diff --git a/ceph-mon/Chart.yaml b/ceph-mon/Chart.yaml index f4ea833057..4294a495b5 100644 --- a/ceph-mon/Chart.yaml +++ b/ceph-mon/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Mon name: ceph-mon -version: 0.1.29 +version: 0.1.30 home: https://github.com/ceph/ceph ... diff --git a/ceph-mon/templates/daemonset-mon.yaml b/ceph-mon/templates/daemonset-mon.yaml index a7368be01e..1b6e9c9339 100644 --- a/ceph-mon/templates/daemonset-mon.yaml +++ b/ceph-mon/templates/daemonset-mon.yaml @@ -12,6 +12,18 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "monLivenessProbeTemplate" -}} +exec: + command: + - /tmp/mon-check.sh +{{- end -}} + +{{- define "monReadinessProbeTemplate" -}} +exec: + command: + - /tmp/mon-check.sh +{{- end -}} + {{- if and .Values.manifests.daemonset_mon .Values.deployment.ceph }} {{- $envAll := . }} @@ -175,20 +187,8 @@ spec: ports: - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - containerPort: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - livenessProbe: - exec: - command: - - /tmp/mon-check.sh - - liveness - initialDelaySeconds: 360 - periodSeconds: 180 - readinessProbe: - exec: - command: - - /tmp/mon-check.sh - - readiness - initialDelaySeconds: 60 - periodSeconds: 60 +{{ dict "envAll" . "component" "ceph" "container" "ceph-mon" "type" "liveness" "probeTemplate" (include "monLivenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph" "container" "ceph-mon" "type" "readiness" "probeTemplate" (include "monReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} volumeMounts: - name: pod-tmp mountPath: /tmp diff --git a/ceph-mon/templates/deployment-mgr.yaml b/ceph-mon/templates/deployment-mgr.yaml index b544276f70..7f2b4b1233 100644 --- a/ceph-mon/templates/deployment-mgr.yaml +++ b/ceph-mon/templates/deployment-mgr.yaml @@ -12,6 +12,18 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "mgrLivenessProbeTemplate" -}} +exec: + command: + - /tmp/mgr-check.sh +{{- end }} + +{{- define "mgrReadinessProbeTemplate" -}} +exec: + command: + - /tmp/mgr-check.sh +{{- end }} + {{- if and .Values.manifests.deployment_mgr (and .Values.deployment.ceph .Values.conf.features.mgr ) }} {{- $envAll := . }} @@ -126,20 +138,6 @@ spec: - name: metrics containerPort: {{ tuple "ceph_mgr" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} {{ end -}} - livenessProbe: - exec: - command: - - /tmp/mgr-check.sh - - liveness - initialDelaySeconds: 30 - timeoutSeconds: 5 - readinessProbe: - exec: - command: - - /tmp/mgr-check.sh - - readiness - initialDelaySeconds: 30 - timeoutSeconds: 5 volumeMounts: - name: pod-tmp mountPath: /tmp @@ -177,6 +175,8 @@ spec: mountPath: /tmp/utils-checkPGs.py subPath: utils-checkPGs.py readOnly: true +{{ dict "envAll" . "component" "ceph" "container" "ceph-mgr" "type" "liveness" "probeTemplate" (include "mgrLivenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph" "container" "ceph-mgr" "type" "readiness" "probeTemplate" (include "mgrReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} volumes: - name: pod-tmp emptyDir: {} diff --git a/ceph-mon/values.yaml b/ceph-mon/values.yaml index 32e86ca751..c485c115f7 100644 --- a/ceph-mon/values.yaml +++ b/ceph-mon/values.yaml @@ -207,6 +207,32 @@ pod: key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 60 + probes: + ceph: + ceph-mon: + readiness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 360 + periodSeconds: 180 + timeoutSeconds: 5 + ceph-mgr: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 5 secrets: keyrings: diff --git a/ceph-osd/Chart.yaml b/ceph-osd/Chart.yaml index d10448c158..f5a617da42 100644 --- a/ceph-osd/Chart.yaml +++ b/ceph-osd/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph OSD name: ceph-osd -version: 0.1.45 +version: 0.1.46 home: https://github.com/ceph/ceph ... diff --git a/ceph-osd/templates/daemonset-osd.yaml b/ceph-osd/templates/daemonset-osd.yaml index 522f9e60f6..3ba2ce7e99 100644 --- a/ceph-osd/templates/daemonset-osd.yaml +++ b/ceph-osd/templates/daemonset-osd.yaml @@ -12,6 +12,18 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "osdLivenessProbeTemplate" -}} +exec: + command: + - /tmp/osd-check.sh +{{- end -}} + +{{- define "osdReadinessProbeTemplate" -}} +exec: + command: + - /tmp/osd-check.sh +{{- end -}} + {{- if .Values.manifests.daemonset_osd }} {{- $envAll := . }} @@ -352,20 +364,8 @@ spec: exec: command: - /tmp/osd-stop.sh - livenessProbe: - exec: - command: - - /tmp/osd-check.sh - - liveness - initialDelaySeconds: 120 - periodSeconds: 60 - readinessProbe: - exec: - command: - - /tmp/osd-check.sh - - readiness - initialDelaySeconds: 60 - periodSeconds: 60 +{{ dict "envAll" . "component" "ceph-osd" "container" "ceph-osd" "type" "liveness" "probeTemplate" (include "osdLivenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph-osd" "container" "ceph-osd" "type" "readiness" "probeTemplate" (include "osdReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} volumeMounts: - name: pod-tmp mountPath: /tmp diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml index 7fe7770d5e..3179f3a371 100644 --- a/ceph-osd/values.yaml +++ b/ceph-osd/values.yaml @@ -137,6 +137,21 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + probes: + ceph-osd: + ceph-osd: + readiness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 120 + periodSeconds: 60 + timeoutSeconds: 5 secrets: keyrings: diff --git a/releasenotes/notes/ceph-client.yaml b/releasenotes/notes/ceph-client.yaml index 26da7f164d..bddbe9dfeb 100644 --- a/releasenotes/notes/ceph-client.yaml +++ b/releasenotes/notes/ceph-client.yaml @@ -47,4 +47,5 @@ ceph-client: - 0.1.44 Allow pg_num_min to be overridden per pool - 0.1.45 Update Ceph to 17.2.6 - 0.1.46 Strip any errors preceding pool properties JSON + - 0.1.47 Use Helm toolkit functions for Ceph probes ... diff --git a/releasenotes/notes/ceph-mon.yaml b/releasenotes/notes/ceph-mon.yaml index 4c7f327025..e8d4d66999 100644 --- a/releasenotes/notes/ceph-mon.yaml +++ b/releasenotes/notes/ceph-mon.yaml @@ -30,4 +30,5 @@ ceph-mon: - 0.1.27 Update all Ceph images to Focal - 0.1.28 Document the use of mon_allow_pool_size_one - 0.1.29 Update Ceph to 17.2.6 + - 0.1.30 Use Helm tookkit functions for Ceph probes ... diff --git a/releasenotes/notes/ceph-osd.yaml b/releasenotes/notes/ceph-osd.yaml index e4c50b4ed2..a4c5fe6b6e 100644 --- a/releasenotes/notes/ceph-osd.yaml +++ b/releasenotes/notes/ceph-osd.yaml @@ -46,4 +46,5 @@ ceph-osd: - 0.1.43 Update all Ceph images to Focal - 0.1.44 Update Ceph to 17.2.6 - 0.1.45 Extend the ceph-osd post-apply job PG wait + - 0.1.46 Use Helm toolkit functions for Ceph probes ...