From 4a95f75b6bbd84f889fd95056e67f7bdda3965b9 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Fri, 1 Dec 2023 22:09:13 +0000 Subject: [PATCH] [backups] Added staggered backups This PS adds staggered backups possibility by adding anti-affinity rules to backups cronjobs that can be followed across several namespaces to decrease load on remote backup destination server making sure that at every moment in time there is only one backup upload is in progress. Change-Id: If49791f866a73a08fb98fa0e0b4854042d079c66 --- helm-toolkit/Chart.yaml | 2 +- .../db-backup-restore/_backup_main.sh.tpl | 2 +- mariadb-backup/Chart.yaml | 2 +- .../templates/cron-job-backup-mariadb.yaml | 6 +++ .../values_overrides/staggered-backups.yaml | 38 +++++++++++++++++++ mariadb/Chart.yaml | 2 +- .../templates/cron-job-backup-mariadb.yaml | 13 ++++--- mariadb/values_overrides/backups.yaml | 15 ++++++++ .../values_overrides/staggered-backups.yaml | 38 +++++++++++++++++++ postgresql/Chart.yaml | 2 +- .../templates/cron-job-backup-postgres.yaml | 9 +++-- postgresql/values_overrides/backups.yaml | 15 ++++++++ .../values_overrides/staggered-backups.yaml | 38 +++++++++++++++++++ releasenotes/notes/helm-toolkit.yaml | 1 + releasenotes/notes/mariadb-backup.yaml | 1 + releasenotes/notes/mariadb.yaml | 1 + releasenotes/notes/postgresql.yaml | 1 + 17 files changed, 173 insertions(+), 13 deletions(-) create mode 100644 mariadb-backup/values_overrides/staggered-backups.yaml create mode 100644 mariadb/values_overrides/backups.yaml create mode 100644 mariadb/values_overrides/staggered-backups.yaml create mode 100644 postgresql/values_overrides/backups.yaml create mode 100644 postgresql/values_overrides/staggered-backups.yaml diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml index e4b45e31cb..a22b4f9c29 100644 --- a/helm-toolkit/Chart.yaml +++ b/helm-toolkit/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Helm-Toolkit name: helm-toolkit -version: 0.2.56 +version: 0.2.57 home: https://docs.openstack.org/openstack-helm icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png sources: diff --git a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl index 3963bd4056..2634b6da29 100755 --- a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl +++ b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl @@ -243,7 +243,7 @@ send_to_remote_server() { log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values" return 2 fi - rm -rf ${REMOTE_FILE} + rm -f ${REMOTE_FILE} log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully." return 0 diff --git a/mariadb-backup/Chart.yaml b/mariadb-backup/Chart.yaml index a34b23fc26..f98d06d085 100644 --- a/mariadb-backup/Chart.yaml +++ b/mariadb-backup/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v10.6.14 description: OpenStack-Helm MariaDB backups name: mariadb-backup -version: 0.0.1 +version: 0.0.2 home: https://mariadb.com/kb/en/ icon: http://badges.mariadb.org/mariadb-badge-180x60.png sources: diff --git a/mariadb-backup/templates/cron-job-backup-mariadb.yaml b/mariadb-backup/templates/cron-job-backup-mariadb.yaml index 18dd3e0fd4..87fd91c41e 100644 --- a/mariadb-backup/templates/cron-job-backup-mariadb.yaml +++ b/mariadb-backup/templates/cron-job-backup-mariadb.yaml @@ -56,6 +56,12 @@ spec: {{ if $envAll.Values.pod.tolerations.mariadb.enabled }} {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }} {{ end }} +{{- if $envAll.Values.pod.affinity }} +{{- if $envAll.Values.pod.affinity.mariadb_backup }} + affinity: +{{ index $envAll.Values.pod.affinity "mariadb_backup" | toYaml | indent 12}} +{{- end }} +{{- end }} nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: diff --git a/mariadb-backup/values_overrides/staggered-backups.yaml b/mariadb-backup/values_overrides/staggered-backups.yaml new file mode 100644 index 0000000000..03412d748c --- /dev/null +++ b/mariadb-backup/values_overrides/staggered-backups.yaml @@ -0,0 +1,38 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +pod: + labels: + backup: + staggered_backups: enabled + affinity: + mariadb_backup: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: status.phase + operator: NotIn + values: + - Running + - key: staggered-backups + operator: In + values: + - enabled + namespaces: + - openstack + - osh-infra + - ucp + topologyKey: kubernetes.io/os +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/mariadb/Chart.yaml b/mariadb/Chart.yaml index 101e142579..e859734745 100644 --- a/mariadb/Chart.yaml +++ b/mariadb/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v10.6.7 description: OpenStack-Helm MariaDB name: mariadb -version: 0.2.35 +version: 0.2.36 home: https://mariadb.com/kb/en/ icon: http://badges.mariadb.org/mariadb-badge-180x60.png sources: diff --git a/mariadb/templates/cron-job-backup-mariadb.yaml b/mariadb/templates/cron-job-backup-mariadb.yaml index db8c06639c..e6974ef426 100644 --- a/mariadb/templates/cron-job-backup-mariadb.yaml +++ b/mariadb/templates/cron-job-backup-mariadb.yaml @@ -53,9 +53,15 @@ spec: serviceAccountName: {{ $serviceAccountName }} restartPolicy: OnFailure shareProcessNamespace: true -{{ if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{- if $envAll.Values.pod.tolerations.mariadb.enabled }} {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }} -{{ end }} +{{- end }} +{{- if $envAll.Values.pod.affinity }} +{{- if $envAll.Values.pod.affinity.mariadb_backup }} + affinity: +{{ index $envAll.Values.pod.affinity "mariadb_backup" | toYaml | indent 12}} +{{- end }} +{{- end }} nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: @@ -191,9 +197,6 @@ spec: mountPath: /tmp/start_verification_server.sh readOnly: true subPath: start_verification_server.sh - restartPolicy: OnFailure - serviceAccount: {{ $serviceAccountName }} - serviceAccountName: {{ $serviceAccountName }} volumes: - name: pod-tmp emptyDir: {} diff --git a/mariadb/values_overrides/backups.yaml b/mariadb/values_overrides/backups.yaml new file mode 100644 index 0000000000..5a7de206c1 --- /dev/null +++ b/mariadb/values_overrides/backups.yaml @@ -0,0 +1,15 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/mariadb/values_overrides/staggered-backups.yaml b/mariadb/values_overrides/staggered-backups.yaml new file mode 100644 index 0000000000..03412d748c --- /dev/null +++ b/mariadb/values_overrides/staggered-backups.yaml @@ -0,0 +1,38 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +pod: + labels: + backup: + staggered_backups: enabled + affinity: + mariadb_backup: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: status.phase + operator: NotIn + values: + - Running + - key: staggered-backups + operator: In + values: + - enabled + namespaces: + - openstack + - osh-infra + - ucp + topologyKey: kubernetes.io/os +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/postgresql/Chart.yaml b/postgresql/Chart.yaml index b9ee4aa1cb..a5443202b5 100644 --- a/postgresql/Chart.yaml +++ b/postgresql/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v14.5 description: OpenStack-Helm PostgreSQL name: postgresql -version: 0.1.18 +version: 0.1.19 home: https://www.postgresql.org sources: - https://github.com/postgres/postgres diff --git a/postgresql/templates/cron-job-backup-postgres.yaml b/postgresql/templates/cron-job-backup-postgres.yaml index 3d9394d456..c2e2e8d26d 100644 --- a/postgresql/templates/cron-job-backup-postgres.yaml +++ b/postgresql/templates/cron-job-backup-postgres.yaml @@ -52,6 +52,12 @@ spec: {{ dict "envAll" $envAll "application" "postgresql_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} serviceAccountName: {{ $serviceAccountName }} restartPolicy: OnFailure +{{- if $envAll.Values.pod.affinity }} +{{- if $envAll.Values.pod.affinity.postgresql_backup }} + affinity: +{{ index $envAll.Values.pod.affinity "postgresql_backup" | toYaml | indent 12}} +{{- end }} +{{- end }} nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: @@ -137,9 +143,6 @@ spec: mountPath: /etc/postgresql/admin_user.conf subPath: admin_user.conf readOnly: true - restartPolicy: OnFailure - serviceAccount: {{ $serviceAccountName }} - serviceAccountName: {{ $serviceAccountName }} volumes: - name: pod-tmp emptyDir: {} diff --git a/postgresql/values_overrides/backups.yaml b/postgresql/values_overrides/backups.yaml new file mode 100644 index 0000000000..499322a810 --- /dev/null +++ b/postgresql/values_overrides/backups.yaml @@ -0,0 +1,15 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_postgresql_backup: true + secret_backup_restore: true +... diff --git a/postgresql/values_overrides/staggered-backups.yaml b/postgresql/values_overrides/staggered-backups.yaml new file mode 100644 index 0000000000..f51ba78c93 --- /dev/null +++ b/postgresql/values_overrides/staggered-backups.yaml @@ -0,0 +1,38 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +pod: + labels: + backup: + staggered_backups: enabled + affinity: + postgresql_backup: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: status.phase + operator: NotIn + values: + - Running + - key: staggered-backups + operator: In + values: + - enabled + namespaces: + - openstack + - osh-infra + - ucp + topologyKey: kubernetes.io/os +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_postgresql_backup: true + secret_backup_restore: true +... diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml index 44e26149d3..0df02e72bf 100644 --- a/releasenotes/notes/helm-toolkit.yaml +++ b/releasenotes/notes/helm-toolkit.yaml @@ -63,4 +63,5 @@ helm-toolkit: - 0.2.54 Fix dependency resolver to ignore non-existing dependencyKey when dependencyMixinParam is a slice - 0.2.55 Updated deprecated IngressClass annotation - 0.2.56 Expose S3 credentials from Rook bucket CRD secret + - 0.2.57 Safer file removal ... diff --git a/releasenotes/notes/mariadb-backup.yaml b/releasenotes/notes/mariadb-backup.yaml index e8bc615c63..192fa1d398 100644 --- a/releasenotes/notes/mariadb-backup.yaml +++ b/releasenotes/notes/mariadb-backup.yaml @@ -1,4 +1,5 @@ --- mariadb-backup: - 0.0.1 Initial Chart + - 0.0.2 Added staggered backups support ... diff --git a/releasenotes/notes/mariadb.yaml b/releasenotes/notes/mariadb.yaml index d7ac44994d..4f745ab911 100644 --- a/releasenotes/notes/mariadb.yaml +++ b/releasenotes/notes/mariadb.yaml @@ -51,4 +51,5 @@ mariadb: - 0.2.33 Add 2023.1 Ubuntu Focal overrides - 0.2.34 Uplift ingress controller image to 1.8.2 - 0.2.35 Update apparmor override + - 0.2.36 Added staggered backups support ... diff --git a/releasenotes/notes/postgresql.yaml b/releasenotes/notes/postgresql.yaml index 85bec52b47..9a8368448e 100644 --- a/releasenotes/notes/postgresql.yaml +++ b/releasenotes/notes/postgresql.yaml @@ -19,4 +19,5 @@ postgresql: - 0.1.16 Added OCI registry authentication - 0.1.17 Added empty verify_databases_backup_archives() function implementation to match updated backup_databases() function in helm-toolkit - 0.1.18 Updated postgres to 14.5 and replaced deprecated config item wal_keep_segments with wal_keep_size + - 0.1.19 Added staggered backups support ...