From d58e048aea5ec70f830f1703245b811d1ee54a7b Mon Sep 17 00:00:00 2001 From: Hediberto Cavalcante da Silva Date: Thu, 3 Nov 2022 19:54:49 -0300 Subject: [PATCH] ceph-csi-rbd: add storage-init.yaml Signed-off-by: Hediberto Cavalcante da Silva --- .../ceph-csi-rbd/templates/storage-init.yaml | 279 ++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 charts/ceph-csi-rbd/templates/storage-init.yaml diff --git a/charts/ceph-csi-rbd/templates/storage-init.yaml b/charts/ceph-csi-rbd/templates/storage-init.yaml new file mode 100644 index 0000000..8e8c4de --- /dev/null +++ b/charts/ceph-csi-rbd/templates/storage-init.yaml @@ -0,0 +1,279 @@ +{{/* +# +# Copyright (c) 2020-2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +*/}} + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-rbac-secrets-namespaces + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + "meta.helm.sh/release-name": {{ .Release.Name }} + "meta.helm.sh/release-namespace": {{ .Release.Namespace }} + "helm.sh/hook": "pre-upgrade, pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "create", "list", "update"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-rbac-secrets-namespaces + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + "meta.helm.sh/release-name": {{ .Release.Name }} + "meta.helm.sh/release-namespace": {{ .Release.Namespace }} + "helm.sh/hook": "pre-upgrade, pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: rbd-rbac-secrets-namespaces + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: rbd-storage-init + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + "meta.helm.sh/release-name": {{ .Release.Name }} + "meta.helm.sh/release-namespace": {{ .Release.Namespace }} + "helm.sh/hook": "pre-upgrade, pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" +data: + ceph.conf: | + # + # Copyright (c) 2020-2022 Wind River Systems, Inc. + # + # SPDX-License-Identifier: Apache-2.0 + # + + [global] + # For version 0.55 and beyond, you must explicitly enable + # or disable authentication with "auth" entries in [global]. + auth_cluster_required = none + auth_service_required = none + auth_client_required = none + + {{ $monitors := .Values.classDefaults.monitors }} + {{ range $index, $monitor := $monitors}} + [mon.{{- $index }}] + mon_addr = {{ $monitor }} + {{- end }} + + storage-init.sh: | + # + # Copyright (c) 2020-2022 Wind River Systems, Inc. + # + # SPDX-License-Identifier: Apache-2.0 + # + + #! /bin/bash + + # Copy from read only mount to Ceph config folder + cp /tmp/ceph.conf /etc/ceph/ + + if [ -n "${CEPH_ADMIN_SECRET}" ]; then + kubectl get secret -n ${NAMESPACE} | grep ${CEPH_ADMIN_SECRET} + if [ $? -ne 0 ]; then + echo "Create ${CEPH_ADMIN_SECRET} secret" + kubectl create secret generic ${CEPH_ADMIN_SECRET} --type="kubernetes.io/rbd" --from-literal=key= --namespace=${NAMESPACE} + if [ $? -ne 0 ]; then + echo "Error creating secret ${CEPH_ADMIN_SECRET}, exit" + exit 1 + fi + fi + fi + + touch /etc/ceph/ceph.client.admin.keyring + + # Check if ceph is accessible + echo "====================================" + ceph -s + if [ $? -ne 0 ]; then + echo "Error: Ceph cluster is not accessible, check Pod logs for details." + exit 1 + fi + + set -ex + # Make sure the pool exists. + ceph osd pool stats ${POOL_NAME} || ceph osd pool create ${POOL_NAME} ${POOL_CHUNK_SIZE} + # Set pool configuration. + ceph osd pool application enable ${POOL_NAME} rbd + ceph osd pool set ${POOL_NAME} size ${POOL_REPLICATION} + ceph osd pool set ${POOL_NAME} crush_rule ${POOL_CRUSH_RULE_NAME} + set +ex + + if [[ -z "${USER_ID}" && -z "${CEPH_USER_SECRET}" ]]; then + echo "No need to create secrets for pool ${POOL_NAME}" + exit 0 + fi + + set -ex + KEYRING=$(ceph auth get-or-create client.${USER_ID} mon "allow r" osd "allow rwx pool=${POOL_NAME}" | sed -n 's/^[[:blank:]]*key[[:blank:]]\+=[[:blank:]]\(.*\)/\1/p') + # Set up pool key in Ceph format + CEPH_USER_KEYRING=/etc/ceph/ceph.client.${USER_ID}.keyring + echo $KEYRING > $CEPH_USER_KEYRING + set +ex + + if [ -n "${CEPH_USER_SECRET}" ]; then + kubectl get secret -n ${NAMESPACE} ${CEPH_USER_SECRET} 2>/dev/null + if [ $? -ne 0 ]; then + echo "Create ${CEPH_USER_SECRET} secret" + kubectl create secret generic -n ${NAMESPACE} ${CEPH_USER_SECRET} --type="kubernetes.io/rbd" --from-literal=key=$KEYRING + if [ $? -ne 0 ]; then + echo"Error creating secret ${CEPH_USER_SECRET} in ${NAMESPACE}, exit" + exit 1 + fi + else + echo "Secret ${CEPH_USER_SECRET} already exists" + fi + + # Support creating namespaces and Ceph user secrets for additional + # namespaces other than that which the provisioner is installed. This + # allows the provisioner to set up and provide PVs for multiple + # applications across many namespaces. + if [ -n "${ADDITIONAL_NAMESPACES}" ]; then + for ns in $(IFS=,; echo ${ADDITIONAL_NAMESPACES}); do + kubectl get namespace $ns 2>/dev/null + if [ $? -ne 0 ]; then + kubectl create namespace $ns + if [ $? -ne 0 ]; then + echo "Error creating namespace $ns, exit" + continue + fi + fi + + kubectl get secret -n $ns ${CEPH_USER_SECRET} 2>/dev/null + if [ $? -ne 0 ]; then + echo "Creating secret ${CEPH_USER_SECRET} for namespace $ns" + kubectl create secret generic -n $ns ${CEPH_USER_SECRET} --type="kubernetes.io/rbd" --from-literal=key=$KEYRING + if [ $? -ne 0 ]; then + echo "Error creating secret ${CEPH_USER_SECRET} in $ns, exit" + fi + else + echo "Secret ${CEPH_USER_SECRET} for namespace $ns already exists" + fi + done + fi + fi + + # Check if pool is accessible using provided credentials + echo "=====================================" + timeout --preserve-status 10 rbd -p ${POOL_NAME} --user ${USER_ID} ls -K $CEPH_USER_KEYRING + if [ $? -ne 143 ]; then + if [ $? -ne 0 ]; then + echo "Error: Ceph pool ${POOL_NAME} is not accessible using credentials for user ${USER_ID}, check Pod logs for details." + exit 1 + else + echo "Pool ${POOL_NAME} accessible" + fi + else + echo "rbd command timed out and was sent a SIGTERM. Make sure OSDs have been provisioned." + fi + + ceph -s + +--- + +apiVersion: batch/v1 +kind: Job +metadata: + name: rbd-storage-init + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + "meta.helm.sh/release-name": {{ .Release.Name }} + "meta.helm.sh/release-namespace": {{ .Release.Namespace }} + "helm.sh/hook": "post-install, pre-upgrade, pre-rollback" + "helm.sh/hook-delete-policy": "before-hook-creation" +spec: + backoffLimit: 5 + activeDeadlineSeconds: 360 + template: + metadata: + name: "{{ .Release.Name }}" + namespace: {{ .Release.Namespace }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{- .Chart.Version }}" + spec: + serviceAccountName: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }} + restartPolicy: OnFailure + volumes: + - name: rbd-storage-init-configmap-volume + configMap: + name: rbd-storage-init + containers: + - name: storage-init-{{- .Values.storageClass.name }} + image: {{ .Values.images.tags.rbd_provisioner_storage_init | quote }} + command: [ "/bin/bash", "/tmp/storage-init.sh" ] + env: + - name: NAMESPACE + value: {{ .Release.Namespace }} + - name: ADDITIONAL_NAMESPACES + value: {{ join "," .Values.storageClass.additionalNamespaces | quote }} + - name: CEPH_ADMIN_SECRET + value: {{ .Values.classDefaults.adminSecretName }} + - name: CEPH_USER_SECRET + value: {{ .Values.storageClass.userSecretName }} + - name: USER_ID + value: {{ .Values.storageClass.userId }} + - name: POOL_NAME + value: {{ .Values.storageClass.pool }} + - name: POOL_REPLICATION + value: {{ .Values.storageClass.replication | quote }} + - name: POOL_CRUSH_RULE_NAME + value: {{ .Values.storageClass.crush_rule_name | quote }} + - name: POOL_CHUNK_SIZE + value: {{ .Values.storageClass.chunk_size | quote }} + volumeMounts: + - name: rbd-storage-init-configmap-volume + mountPath: /tmp +{{- if .Values.provisioner.nodeSelector }} + nodeSelector: +{{ .Values.provisioner.nodeSelector | toYaml | trim | indent 8 }} +{{- end }} +{{- with .Values.provisioner.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} -- 2.17.1