From 8b1f987add87d217f720f505b9f65f80fc8dbd20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20de=20Ara=C3=BAjo=20Cabral?= Date: Sun, 3 Dec 2023 20:21:14 -0300 Subject: [PATCH] Add SnapshotClass Creation for CephFS/RBD via Helm Override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit introduces the capability to create a snapshot class using helm overrides within "cephfs-provisioner" and "rbd-provisioner" charts. By default, upon applying platform-integ-apps the 'snapshotClass.create' field is set to 'false' in both charts, creating the snapshot class(es) after being changed to 'true' via helm overrides and the app reapplied. This enhancement depends on the changes from the review below, where in each installation the CRDs and the snapshot-controller will be created by default when running bootstrap playbook. This way, each CSI app will be able to implement this functionality as support for PVC snapshots will be default during installations. Depends-on: https://review.opendev.org/c/starlingx/ansible-playbooks/+/902811 Test Plan: PASS: Build a new app package with the code changes PASS: Successfully execute upload, apply, remove, and delete operations for 'platform-integ-apps' on both AIO-SX and AIO-DX environments. PASS: Upon initial application apply and update, check with 'helm get values' that the 'snapshotClass.create' field is set to 'false' in both the cephFS and RBD charts. Additionally, confirm on K8s that the snapshotClasses are not created, as expected PASS: Use 'system helm-override-update' to change the 'snapshotClass.create' field to 'true' in both charts. Reapply the app and validate on K8s that the snapshotClasses are indeed created. PASS: After creating the SnapshotClass, take a VolumeSnapshot from an existing PVC, proceed with K8s upgrade (K8s 1.25, 1.26, and 1.27) with CRDs and snapshot controller update, take a VolumeSnapshot from another PVC and verify that all VolumeSnapshots are correct Partial-Bug: 2045897 Change-Id: I47b897d179c4260fad9171586fe2fbd69f7145de Signed-off-by: Gabriel de Araújo Cabral --- ...eph-csi-cephfs-add-storage-init.yaml.patch | 9 +-- ...8-ceph-csi-rbd-add-storage-init.yaml.patch | 9 +-- ...011-ceph-csi-rbd-update-storageclass.patch | 9 +-- ...-ceph-csi-cephfs-update-storageclass.patch | 9 +-- ...13-ceph-csi-cephfs-add-snapshotclass.patch | 62 +++++++++++++++++++ .../0014-ceph-csi-rbd-add-snapshotclass.patch | 62 +++++++++++++++++++ .../debian/deb_folder/patches/series | 2 + .../helm/ceph_fs_provisioner.py | 10 ++- .../k8sapp_platform/helm/rbd_provisioner.py | 10 ++- .../cephfs-provisioner-static-overrides.yaml | 6 ++ .../rbd-provisioner-static-overrides.yaml | 6 ++ 11 files changed, 174 insertions(+), 20 deletions(-) create mode 100644 platform-helm/debian/deb_folder/patches/0013-ceph-csi-cephfs-add-snapshotclass.patch create mode 100644 platform-helm/debian/deb_folder/patches/0014-ceph-csi-rbd-add-snapshotclass.patch diff --git a/platform-helm/debian/deb_folder/patches/0003-ceph-csi-cephfs-add-storage-init.yaml.patch b/platform-helm/debian/deb_folder/patches/0003-ceph-csi-cephfs-add-storage-init.yaml.patch index ed70de0..ba0733b 100644 --- a/platform-helm/debian/deb_folder/patches/0003-ceph-csi-cephfs-add-storage-init.yaml.patch +++ b/platform-helm/debian/deb_folder/patches/0003-ceph-csi-cephfs-add-storage-init.yaml.patch @@ -1,11 +1,12 @@ -From 239cfdde94db341aad0ccb15e420c4ecc3157d21 Mon Sep 17 00:00:00 2001 -From: Erickson Silva de Oliveira -Date: Mon, 24 Apr 2023 16:57:46 -0300 +From 356ddba07d33e333d332415d9858f4c6354a0d85 Mon Sep 17 00:00:00 2001 +From: Gabriel de Araújo Cabral +Date: Wed, 6 Dec 2023 14:16:19 -0300 Subject: [PATCH] ceph-csi-cephfs: add storage-init.yaml Signed-off-by: Hediberto Cavalcante da Silva Signed-off-by: Felipe Sanches Zanoni Signed-off-by: Erickson Silva de Oliveira +Signed-off-by: Gabriel de Araújo Cabral --- .../templates/storage-init.yaml | 369 ++++++++++++++++++ 1 file changed, 369 insertions(+) @@ -344,7 +345,7 @@ index 000000000..97052d7a9 + name: cephfs-storage-init + defaultMode: 0555 + containers: -+ {{- range $sc := .Values.classes }} ++ {{- range $sc := .Values.storageClasses }} + - name: storage-init-{{- $sc.name }} + image: "{{ $.Values.storage_init.image.repository }}:{{ $.Values.storage_init.image.tag }}" + command: ["/bin/bash", "/tmp/storage-init.sh"] diff --git a/platform-helm/debian/deb_folder/patches/0008-ceph-csi-rbd-add-storage-init.yaml.patch b/platform-helm/debian/deb_folder/patches/0008-ceph-csi-rbd-add-storage-init.yaml.patch index e4b9d8f..43c430d 100644 --- a/platform-helm/debian/deb_folder/patches/0008-ceph-csi-rbd-add-storage-init.yaml.patch +++ b/platform-helm/debian/deb_folder/patches/0008-ceph-csi-rbd-add-storage-init.yaml.patch @@ -1,11 +1,12 @@ -From 4e3d4d76c16c4d03ba33afd0581ceac5f5ea3147 Mon Sep 17 00:00:00 2001 -From: Erickson Silva de Oliveira -Date: Mon, 24 Apr 2023 17:04:02 -0300 +From 080cad7da551b36ee22139c558770cf6520a0d55 Mon Sep 17 00:00:00 2001 +From: Gabriel de Araújo Cabral +Date: Wed, 6 Dec 2023 14:17:19 -0300 Subject: [PATCH] ceph-csi-rbd: add storage-init.yaml Signed-off-by: Hediberto Cavalcante da Silva Signed-off-by: Felipe Sanches Zanoni Signed-off-by: Erickson Silva de Oliveira +Signed-off-by: Gabriel de Araújo Cabral --- .../ceph-csi-rbd/templates/storage-init.yaml | 394 ++++++++++++++++++ 1 file changed, 394 insertions(+) @@ -372,7 +373,7 @@ index 000000000..6abd8ec3f + configMap: + name: rbd-storage-init + containers: -+ {{- range $sc := .Values.classes }} ++ {{- range $sc := .Values.storageClasses }} + - name: storage-init-{{- $sc.name }} + image: "{{ $.Values.storage_init.image.repository }}:{{ $.Values.storage_init.image.tag }}" + command: [ "/bin/bash", "/tmp/storage-init.sh" ] diff --git a/platform-helm/debian/deb_folder/patches/0011-ceph-csi-rbd-update-storageclass.patch b/platform-helm/debian/deb_folder/patches/0011-ceph-csi-rbd-update-storageclass.patch index 3994b7c..6b0238a 100644 --- a/platform-helm/debian/deb_folder/patches/0011-ceph-csi-rbd-update-storageclass.patch +++ b/platform-helm/debian/deb_folder/patches/0011-ceph-csi-rbd-update-storageclass.patch @@ -1,10 +1,11 @@ -From e02f28d37a57550e4e4127c5b3758e865cfc4d7e Mon Sep 17 00:00:00 2001 -From: Erickson Silva de Oliveira -Date: Mon, 20 Mar 2023 17:56:25 -0300 +From d4ecc92cf187225ab4cdb882370647cb5b5d0d3b Mon Sep 17 00:00:00 2001 +From: Gabriel de Araújo Cabral +Date: Wed, 6 Dec 2023 14:22:19 -0300 Subject: [PATCH] ceph-csi-rbd: update storageclass Signed-off-by: Hediberto Cavalcante da Silva Signed-off-by: Erickson Silva de Oliveira +Signed-off-by: Gabriel de Araújo Cabral --- .../ceph-csi-rbd/templates/storageclass.yaml | 162 ++++++++++-------- 1 file changed, 93 insertions(+), 69 deletions(-) @@ -17,7 +18,7 @@ index a559456aa..b53c3dc6c 100644 {{- if .Values.storageClass.create -}} +{{- $cephCsiDefault := .Values.storageClass }} +{{- $defaults := .Values.classdefaults }} -+{{- range $sc := .Values.classes }} ++{{- range $sc := .Values.storageClasses }} apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/platform-helm/debian/deb_folder/patches/0012-ceph-csi-cephfs-update-storageclass.patch b/platform-helm/debian/deb_folder/patches/0012-ceph-csi-cephfs-update-storageclass.patch index de2b2d6..698d16c 100644 --- a/platform-helm/debian/deb_folder/patches/0012-ceph-csi-cephfs-update-storageclass.patch +++ b/platform-helm/debian/deb_folder/patches/0012-ceph-csi-cephfs-update-storageclass.patch @@ -1,10 +1,11 @@ -From 8ef1e051d4166dd0f9f8c931465b27cadfde7ca0 Mon Sep 17 00:00:00 2001 -From: Erickson Silva de Oliveira -Date: Tue, 21 Mar 2023 21:09:29 -0300 +From 93ae57852cf8583228e99465c3d35e5304b88e65 Mon Sep 17 00:00:00 2001 +From: Gabriel de Araújo Cabral +Date: Wed, 6 Dec 2023 14:24:22 -0300 Subject: [PATCH] ceph-csi-cephfs: update storageclass Signed-off-by: Hediberto Cavalcante da Silva Signed-off-by: Erickson Silva de Oliveira +Signed-off-by: Gabriel de Araújo Cabral --- .../templates/storageclass.yaml | 104 ++++++++++-------- 1 file changed, 56 insertions(+), 48 deletions(-) @@ -17,7 +18,7 @@ index a21c99824..e3a27b455 100644 {{- if .Values.storageClass.create -}} +{{- $cephCsiDefault := .Values.storageClass }} +{{- $defaults := .Values.classdefaults }} -+{{- range $sc := .Values.classes }} ++{{- range $sc := .Values.storageClasses }} apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/platform-helm/debian/deb_folder/patches/0013-ceph-csi-cephfs-add-snapshotclass.patch b/platform-helm/debian/deb_folder/patches/0013-ceph-csi-cephfs-add-snapshotclass.patch new file mode 100644 index 0000000..e67680f --- /dev/null +++ b/platform-helm/debian/deb_folder/patches/0013-ceph-csi-cephfs-add-snapshotclass.patch @@ -0,0 +1,62 @@ +From cc40abf2495bdaa0ace32441cfc6d5a78a397d3b Mon Sep 17 00:00:00 2001 +From: Gabriel de Araújo Cabral +Date: Thu, 30 Nov 2023 12:51:37 -0300 +Subject: ceph-csi-cephfs: add snapshotclass + +Signed-off-by: Gabriel de Araújo Cabral +--- + .../templates/snapshotclass.yaml | 13 +++++++++++++ + charts/ceph-csi-cephfs/values.yaml | 18 ++++++++++++++++++ + 2 files changed, 31 insertions(+) + create mode 100644 charts/ceph-csi-cephfs/templates/snapshotclass.yaml + +diff --git a/charts/ceph-csi-cephfs/templates/snapshotclass.yaml b/charts/ceph-csi-cephfs/templates/snapshotclass.yaml +new file mode 100644 +index 000000000..3bad19026 +--- /dev/null ++++ b/charts/ceph-csi-cephfs/templates/snapshotclass.yaml +@@ -0,0 +1,13 @@ ++{{- if .Values.snapshotClass.create -}} ++apiVersion: snapshot.storage.k8s.io/v1 ++kind: VolumeSnapshotClass ++metadata: ++ name: {{ .Values.snapshotClass.name }} ++driver: {{ .Values.driverName }} ++parameters: ++ clusterID: {{ .Values.snapshotClass.clusterID }} ++ snapshotNamePrefix: {{ .Values.snapshotClass.snapshotNamePrefix }} ++ csi.storage.k8s.io/snapshotter-secret-name: {{ .Values.snapshotClass.provisionerSecret }} ++ csi.storage.k8s.io/snapshotter-secret-namespace: {{ .Values.snapshotClass.provisionerSecretNamespace }} ++deletionPolicy: {{ .Values.snapshotClass.deletionPolicy }} ++{{- end -}} +diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml +index 7c85abd7d..912165f1d 100644 +--- a/charts/ceph-csi-cephfs/values.yaml ++++ b/charts/ceph-csi-cephfs/values.yaml +@@ -302,6 +302,24 @@ storageClass: + # Ceph pools name + metadata_pool: kube-cephfs-metadata + ++snapshotClass: ++ # Specifies whether the snapshotclass should be created ++ create: false ++ # Specifies the snapshotclass name ++ name: csi-cephfsplugin-snapclass ++ # (required) Unique string representing a Ceph cluster to provision ++ #storage from. ++ clusterID: ++ # Prefix to use for naming CephFS snapshots. ++ # If omitted, defaults to "csi-snap-". ++ # snapshotNamePrefix: "foo-bar-" ++ snapshotNamePrefix: "" ++ # K8 secret name ++ provisionerSecret: csi-cephfs-secret ++ # K8 secret namespace ++ provisionerSecretNamespace: "" ++ deletionPolicy: Delete ++ + secret: + # Specifies whether the secret should be created + create: false +-- +2.25.1 diff --git a/platform-helm/debian/deb_folder/patches/0014-ceph-csi-rbd-add-snapshotclass.patch b/platform-helm/debian/deb_folder/patches/0014-ceph-csi-rbd-add-snapshotclass.patch new file mode 100644 index 0000000..8cfc965 --- /dev/null +++ b/platform-helm/debian/deb_folder/patches/0014-ceph-csi-rbd-add-snapshotclass.patch @@ -0,0 +1,62 @@ +From d6dbc163f3be00789a74f85556de4486792f9d72 Mon Sep 17 00:00:00 2001 +From: Gabriel de Araújo Cabral +Date: Thu, 30 Nov 2023 13:38:09 -0300 +Subject: ceph-csi-rbd: add snapshotclass + +Signed-off-by: Gabriel de Araújo Cabral +--- + .../ceph-csi-rbd/templates/snapshotclass.yaml | 13 +++++++++++++ + charts/ceph-csi-rbd/values.yaml | 18 ++++++++++++++++++ + 2 files changed, 31 insertions(+) + create mode 100644 charts/ceph-csi-rbd/templates/snapshotclass.yaml + +diff --git a/charts/ceph-csi-rbd/templates/snapshotclass.yaml b/charts/ceph-csi-rbd/templates/snapshotclass.yaml +new file mode 100644 +index 000000000..3bad19026 +--- /dev/null ++++ b/charts/ceph-csi-rbd/templates/snapshotclass.yaml +@@ -0,0 +1,13 @@ ++{{- if .Values.snapshotClass.create -}} ++apiVersion: snapshot.storage.k8s.io/v1 ++kind: VolumeSnapshotClass ++metadata: ++ name: {{ .Values.snapshotClass.name }} ++driver: {{ .Values.driverName }} ++parameters: ++ clusterID: {{ .Values.snapshotClass.clusterID }} ++ snapshotNamePrefix: {{ .Values.snapshotClass.snapshotNamePrefix }} ++ csi.storage.k8s.io/snapshotter-secret-name: {{ .Values.snapshotClass.provisionerSecret }} ++ csi.storage.k8s.io/snapshotter-secret-namespace: {{ .Values.snapshotClass.provisionerSecretNamespace }} ++deletionPolicy: {{ .Values.snapshotClass.deletionPolicy }} ++{{- end -}} +diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml +index c33d5f2a6..8dd9ea3d5 100644 +--- a/charts/ceph-csi-rbd/values.yaml ++++ b/charts/ceph-csi-rbd/values.yaml +@@ -449,6 +449,24 @@ storageClass: + # selinux-enabled filesystems + selinuxMount: true + ++snapshotClass: ++ # Specifies whether the snapshotclass should be created ++ create: false ++ # Specifies the snapshotclass name ++ name: csi-rbdplugin-snapclass ++ # (required) Unique string representing a Ceph cluster to provision ++ #storage from. ++ clusterID: ++ # Prefix to use for naming RBD snapshots. ++ # If omitted, defaults to "csi-snap-". ++ # snapshotNamePrefix: "foo-bar-" ++ snapshotNamePrefix: "" ++ # K8 secret name ++ provisionerSecret: csi-rbd-secret ++ # K8 secret namespace ++ provisionerSecretNamespace: "" ++ deletionPolicy: Delete ++ + secret: + # Specifies whether the secret should be created + create: false +-- +2.25.1 diff --git a/platform-helm/debian/deb_folder/patches/series b/platform-helm/debian/deb_folder/patches/series index 4990684..f1a0307 100644 --- a/platform-helm/debian/deb_folder/patches/series +++ b/platform-helm/debian/deb_folder/patches/series @@ -10,3 +10,5 @@ 0010-ceph-csi-rbd-add-annotations-to-provisioner-deployment.patch 0011-ceph-csi-rbd-update-storageclass.patch 0012-ceph-csi-cephfs-update-storageclass.patch +0013-ceph-csi-cephfs-add-snapshotclass.patch +0014-ceph-csi-rbd-add-snapshotclass.patch diff --git a/python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/helm/ceph_fs_provisioner.py b/python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/helm/ceph_fs_provisioner.py index 7fa4711..895daee 100644 --- a/python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/helm/ceph_fs_provisioner.py +++ b/python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/helm/ceph_fs_provisioner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2022 Wind River Systems, Inc. +# Copyright (c) 2020-2023 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -192,6 +192,11 @@ class CephFSProvisionerHelm(base.FluxCDBaseHelm): storage_classes.append(storage_class) + snapshot_class = { + "clusterID": cluster_id, + "provisionerSecret": user_secret_name or class_defaults["adminSecretName"] + } + provisioner = { "replicaCount": self._num_replicas_for_platform_app() } @@ -206,7 +211,8 @@ class CephFSProvisionerHelm(base.FluxCDBaseHelm): overrides = { app_constants.HELM_NS_CEPH_FS_PROVISIONER: { - "classes": storage_classes, + "storageClasses": storage_classes, + "snapshotClass": snapshot_class, "provisioner": provisioner, "csiConfig": csi_config, "classdefaults": class_defaults diff --git a/python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/helm/rbd_provisioner.py b/python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/helm/rbd_provisioner.py index 8083c6c..77527ac 100644 --- a/python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/helm/rbd_provisioner.py +++ b/python3-k8sapp-platform/k8sapp_platform/k8sapp_platform/helm/rbd_provisioner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2022 Wind River Systems, Inc. +# Copyright (c) 2020-2023 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -107,6 +107,11 @@ class RbdProvisionerHelm(base.FluxCDBaseHelm): storage_classes.append(storage_class) + snapshot_class = { + "clusterID": cluster_id, + "provisionerSecret": user_secret_name or class_defaults["adminSecretName"] + } + provisioner = { "replicaCount": self._num_replicas_for_platform_app() } @@ -118,7 +123,8 @@ class RbdProvisionerHelm(base.FluxCDBaseHelm): overrides = { common.HELM_NS_RBD_PROVISIONER: { - "classes": storage_classes, + "storageClasses": storage_classes, + "snapshotClass": snapshot_class, "provisioner": provisioner, "csiConfig": csi_config, "classdefaults": class_defaults diff --git a/stx-platform-helm/stx-platform-helm/fluxcd-manifests/cephfs-provisioner/cephfs-provisioner-static-overrides.yaml b/stx-platform-helm/stx-platform-helm/fluxcd-manifests/cephfs-provisioner/cephfs-provisioner-static-overrides.yaml index 6627683..139c8d2 100644 --- a/stx-platform-helm/stx-platform-helm/fluxcd-manifests/cephfs-provisioner/cephfs-provisioner-static-overrides.yaml +++ b/stx-platform-helm/stx-platform-helm/fluxcd-manifests/cephfs-provisioner/cephfs-provisioner-static-overrides.yaml @@ -83,6 +83,12 @@ storageClass: "helm.sh/hook-delete-policy": "before-hook-creation" } +snapshotClass: + create: false + name: cephfs-snapshot + snapshotNamePrefix: cephfs-snap- + provisionerSecretNamespace: kube-system + topology: enabled: false diff --git a/stx-platform-helm/stx-platform-helm/fluxcd-manifests/rbd-provisioner/rbd-provisioner-static-overrides.yaml b/stx-platform-helm/stx-platform-helm/fluxcd-manifests/rbd-provisioner/rbd-provisioner-static-overrides.yaml index 52ba7c6..f84e4d9 100644 --- a/stx-platform-helm/stx-platform-helm/fluxcd-manifests/rbd-provisioner/rbd-provisioner-static-overrides.yaml +++ b/stx-platform-helm/stx-platform-helm/fluxcd-manifests/rbd-provisioner/rbd-provisioner-static-overrides.yaml @@ -84,6 +84,12 @@ storageClass: mountOptions: - discard +snapshotClass: + create: false + name: rbd-snapshot + snapshotNamePrefix: rbd-snap- + provisionerSecretNamespace: kube-system + topology: enabled: false