Merge "Add backup/restore capability to Postgresql utility deployment"

2020-06-22 16:06:12 +00:00 · 2020-06-22 16:06:12 +00:00 · 1643b82d37
commit 1643b82d37
parent 1ac9f7ae2a bfe30402dd
6 changed files with 501 additions and 15 deletions
--- a/charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl
+++ b/charts/postgresql-utility/templates/bin/utility/_pg_ondemand_job.sh.tpl
@ -0,0 +1,182 @@
+#!/bin/bash
+
+{{- $envAll := . }}
+
+export POSTGRESQL_POD_NAMESPACE=$1
+if [[ $POSTGRESQL_POD_NAMESPACE == "" ]]; then
+  echo "No namespace given - cannot spawn ondemand job."
+  exit 1
+fi
+
+export POSTGRESQL_RGW_SECRET={{ $envAll.Values.conf.postgresql_backup_restore.secrets.rgw_secret }}
+export POSTGRESQL_CONF_SECRET={{ $envAll.Values.conf.postgresql_backup_restore.secrets.conf_secret }}
+export POSTGRESQL_IMAGE_NAME=$(kubectl get cronjob -n ucp postgresql-backup -o yaml -o jsonpath="{range .spec.jobTemplate.spec.template.spec.containers[*]}{.image}{'\n'}{end}" | grep postgresql-utility)
+export POSTGRESQL_BACKUP_BASE_PATH=$(kubectl get secret -o yaml -n ${POSTGRESQL_POD_NAMESPACE} ${POSTGRESQL_CONF_SECRET} | grep BACKUP_BASE_PATH | awk '{print $2}' | base64 -d)
+
+if [[ $POSTGRESQL_IMAGE_NAME == "" ]]; then
+  echo "Cannot find the utility image for populating POSTGRESQL_IMAGE_NAME variable."
+  exit 1
+fi
+
+cat <<EOF | kubectl create -n $POSTGRESQL_POD_NAMESPACE -f -
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: postgresql-ondemand
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "postgresql-ondemand" "ondemand" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "postgresql-ondemand" "ondemand" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "postgresql_ondemand" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.utility.node_selector_key }}: {{ .Values.labels.utility.node_selector_value }}
+      initContainers:
+        - name: backup-perms
+          image: ${POSTGRESQL_IMAGE_NAME}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.postgresql_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "postgresql_ondemand" "container" "backup_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - chown
+            - -R
+            - "65534:65534"
+            - ${POSTGRESQL_BACKUP_BASE_PATH}
+          env:
+            - name: POSTGRESQL_BACKUP_BASE_DIR
+              value: ${POSTGRESQL_BACKUP_BASE_PATH}
+          volumeMounts:
+            - mountPath: /tmp
+              name: pod-tmp
+            - mountPath: ${POSTGRESQL_BACKUP_BASE_PATH}
+              name: postgresql-backup-dir
+      containers:
+        - name: postgresql-ondemand
+          image: ${POSTGRESQL_IMAGE_NAME}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.postgresql_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "postgresql_ondemand" "container" "postgresql_ondemand" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - /bin/sleep
+            - "1000000"
+          env:
+            - name: POSTGRESQL_ADMIN_USER
+              valueFrom:
+                secretKeyRef:
+                  key: POSTGRES_USER
+                  name: postgresql-admin
+            - name: POSTGRESQL_BACKUP_BASE_DIR
+              valueFrom:
+                secretKeyRef:
+                  key: BACKUP_BASE_PATH
+                  name: ${POSTGRESQL_CONF_SECRET}
+            - name: POSTGRESQL_POD_NAMESPACE
+              value: ${POSTGRESQL_POD_NAMESPACE}
+            - name: REMOTE_BACKUP_ENABLED
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_ENABLED
+                  name: ${POSTGRESQL_CONF_SECRET}
+            - name: POSTGRESQL_LOCAL_BACKUP_DAYS_TO_KEEP
+              valueFrom:
+                secretKeyRef:
+                  key: LOCAL_DAYS_TO_KEEP
+                  name: ${POSTGRESQL_CONF_SECRET}
+            - name: POSTGRESQL_REMOTE_BACKUP_DAYS_TO_KEEP
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_DAYS_TO_KEEP
+                  name: ${POSTGRESQL_CONF_SECRET}
+            - name: CONTAINER_NAME
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_CONTAINER
+                  name: ${POSTGRESQL_CONF_SECRET}
+            - name: POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS
+              valueFrom:
+                secretKeyRef:
+                  key: PG_DUMPALL_OPTIONS
+                  name: ${POSTGRESQL_CONF_SECRET}
+            - name: OS_IDENTITY_API_VERSION
+              value: "3"
+            - name: OS_AUTH_URL
+              valueFrom:
+                secretKeyRef:
+                  name: ${POSTGRESQL_RGW_SECRET}
+                  key: OS_AUTH_URL
+            - name: OS_REGION_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${POSTGRESQL_RGW_SECRET}
+                  key: OS_REGION_NAME
+            - name: OS_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${POSTGRESQL_RGW_SECRET}
+                  key: OS_USERNAME
+            - name: OS_PROJECT_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${POSTGRESQL_RGW_SECRET}
+                  key: OS_PROJECT_NAME
+            - name: OS_USER_DOMAIN_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${POSTGRESQL_RGW_SECRET}
+                  key: OS_USER_DOMAIN_NAME
+            - name: OS_PROJECT_DOMAIN_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${POSTGRESQL_RGW_SECRET}
+                  key: OS_PROJECT_DOMAIN_NAME
+            - name: OS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: ${POSTGRESQL_RGW_SECRET}
+                  key: OS_PASSWORD
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - mountPath: /tmp/restore_postgresql.sh
+              name: postgresql-bin
+              readOnly: true
+              subPath: restore_postgresql.sh
+            - mountPath: /tmp/restore_main.sh
+              name: postgresql-bin
+              readOnly: true
+              subPath: restore_main.sh
+            - mountPath: /tmp/backup_postgresql.sh
+              name: postgresql-bin
+              readOnly: true
+              subPath: backup_postgresql.sh
+            - mountPath: /tmp/backup_main.sh
+              name: postgresql-bin
+              readOnly: true
+              subPath: backup_main.sh
+            - mountPath: ${POSTGRESQL_BACKUP_BASE_PATH}
+              name: postgresql-backup-dir
+            - name: postgresql-secrets
+              mountPath: /etc/postgresql/admin_user.conf
+              subPath: admin_user.conf
+              readOnly: true
+      restartPolicy: OnFailure
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: postgresql-secrets
+          secret:
+            secretName: postgresql-secrets
+            defaultMode: 292
+        - name: postgresql-bin
+          secret:
+            secretName: postgresql-bin
+            defaultMode: 365
+        - name: postgresql-backup-dir
+          persistentVolumeClaim:
+            claimName: postgresql-backup-data
+EOF
--- a/charts/postgresql-utility/templates/bin/utility/_pgutils.sh.tpl
+++ b/charts/postgresql-utility/templates/bin/utility/_pgutils.sh.tpl
@ -0,0 +1,221 @@
+#!/bin/bash
+
+function database_cmd() {
+  NAMESPACE=$1
+
+  get_postgres_password() {
+    PW=$(kubectl get secret -n "$NAMESPACE" postgresql-admin -o yaml  | grep POSTGRES_PASSWORD | awk '{print $2}' | base64 -d)
+    echo "$PW"
+  }
+  POSTGRES_PWD=$(get_postgres_password)
+  POSTGRES_CREDS="postgresql://postgres:${POSTGRES_PWD}@postgresql.${NAMESPACE}.svc.cluster.local?sslmode=disable"
+  SQL_CMD="psql $POSTGRES_CREDS"
+
+  echo $SQL_CMD
+}
+
+# Params: <namespace>
+function show_databases() {
+
+  SHOW_ARGS=("$@")
+
+  NAMESPACE=${SHOW_ARGS[1]}
+
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  ${DB_CMD} -c "\l"
+}
+
+# Params: <namespace> <database>
+function show_tables() {
+
+  SHOW_ARGS=("$@")
+
+  NAMESPACE=${SHOW_ARGS[1]}
+  DATABASE=${SHOW_ARGS[2]}
+
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  ${DB_CMD} << EOF
+    \connect ${DATABASE};
+    \dt
+EOF
+}
+
+# Params: <namespace> <database> <table>
+function show_rows() {
+
+  SHOW_ARGS=("$@")
+
+  NAMESPACE=${SHOW_ARGS[1]}
+  DATABASE=${SHOW_ARGS[2]}
+  TABLE=${SHOW_ARGS[3]}
+
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  ${DB_CMD} << EOF
+    \connect ${DATABASE};
+    SELECT * FROM ${TABLE};
+EOF
+}
+
+# Params: <namespace> <database> <table>
+function show_schema() {
+
+  SHOW_ARGS=("$@")
+
+  NAMESPACE=${SHOW_ARGS[1]}
+  DATABASE=${SHOW_ARGS[2]}
+  TABLE=${SHOW_ARGS[3]}
+
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  ${DB_CMD} << EOF
+    \connect ${DATABASE};
+    \d ${TABLE};
+EOF
+}
+
+# Params: <namespace>
+function sql_prompt() {
+
+  SHOW_ARGS=("$@")
+
+  NAMESPACE=${SHOW_ARGS[1]}
+
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  ${DB_CMD}
+}
+
+
+# Params: <namespace> <database>
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification/deletion of
+#         an application database.
+function create_database() {
+
+  CREATE_ARGS=("$@")
+
+  NAMESPACE=${CREATE_ARGS[1]}
+  DATABASE="test_"
+  DATABASE+=${CREATE_ARGS[2]}
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  ${DB_CMD} -c "CREATE DATABASE ${DATABASE};"
+}
+
+# Params: <namespace> <database> <tablename>
+#   Column names and types will be hardcoded for now
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification of
+#         an application database.
+function create_table() {
+
+  CREATE_ARGS=("$@")
+
+  NAMESPACE=${CREATE_ARGS[1]}
+  DATABASE="test_"
+  DATABASE+=${CREATE_ARGS[2]}
+  TABLENAME=${CREATE_ARGS[3]}
+
+  CREATE_CMD="CREATE TABLE ${TABLENAME} ( name character varying (255), age integer NOT NULL )"
+
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  $DB_CMD << EOF
+    \connect ${DATABASE};
+    ${CREATE_CMD};
+EOF
+}
+
+# Params: <namespace> <database> <table>
+#   The row values are hardcoded for now.
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification of
+#         an application database.
+function create_row() {
+
+  CREATE_ARGS=("$@")
+
+  NAMESPACE=${CREATE_ARGS[1]}
+  DATABASE="test_"
+  DATABASE+=${CREATE_ARGS[2]}
+  TABLENAME=${CREATE_ARGS[3]}
+
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  NUMROWS=$(echo '\c '"${DATABASE};"' \\ SELECT count(*) from '"${TABLENAME};" | ${DB_CMD} | sed -n '4p' | awk '{print $1}')
+  NAME="name${NUMROWS}"
+  AGE="${NUMROWS}"
+  INSERT_CMD="INSERT INTO ${TABLENAME} VALUES ( '${NAME}', '${AGE}' )"
+
+  $DB_CMD << EOF
+    \connect ${DATABASE};
+    ${INSERT_CMD};
+EOF
+}
+
+# Params: <namespace> <database> <table> <colname> <value>
+#   Where: <colname> = <value> is the condition used to find the row to be deleted.
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification/deletion of
+#         an application database.
+function delete_row() {
+
+  DELETE_ARGS=("$@")
+
+  NAMESPACE=${DELETE_ARGS[1]}
+  DATABASE="test_"
+  DATABASE+=${DELETE_ARGS[2]}
+  TABLENAME=${DELETE_ARGS[3]}
+  COLNAME=${DELETE_ARGS[4]}
+  VALUE=${DELETE_ARGS[5]}
+
+  DELETE_CMD="DELETE FROM ${TABLENAME} WHERE ${COLNAME} = '${VALUE}'"
+
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  ${DB_CMD} << EOF
+    \connect ${DATABASE};
+    ${DELETE_CMD};
+EOF
+}
+
+# Params: <namespace> <database> <tablename>
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification/deletion of
+#         an application database.
+function delete_table() {
+
+  DELETE_ARGS=("$@")
+
+  NAMESPACE=${DELETE_ARGS[1]}
+  DATABASE="test_"
+  DATABASE+=${DELETE_ARGS[2]}
+  TABLENAME=${DELETE_ARGS[3]}
+
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  ${DB_CMD} << EOF
+    \connect ${DATABASE};
+    DROP TABLE IF EXISTS ${TABLENAME};
+EOF
+}
+
+# Params: <namespace> <database>
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification/deletion of
+#         an application database.
+function delete_database() {
+
+  DELETE_ARGS=("$@")
+
+  NAMESPACE=${DELETE_ARGS[1]}
+  DATABASE="test_"
+  DATABASE+=${DELETE_ARGS[2]}
+
+  DB_CMD=$(database_cmd $NAMESPACE)
+
+  ${DB_CMD} -c "DROP DATABASE IF EXISTS ${DATABASE};"
+}
--- a/charts/postgresql-utility/templates/configmap-bin.yaml
+++ b/charts/postgresql-utility/templates/configmap-bin.yaml
@ -25,4 +25,13 @@ data:
  utilscli: |
 {{ tuple "bin/utility/_utilscli.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}

+  dbutils: |
+{{ tuple "bin/utility/_dbutils.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
+  pgutils.sh: |
+{{ tuple "bin/utility/_pgutils.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
+  pg-ondemand-job.sh: |
+{{ tuple "bin/utility/_pg_ondemand_job.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
 {{- end }}
--- a/charts/postgresql-utility/templates/deployment-utility.yaml
+++ b/charts/postgresql-utility/templates/deployment-utility.yaml
@ -28,10 +28,37 @@ rules:
      - namespaces
      - persistentvolumeclaims
      - persistentvolumes
+      - pods
+      - secrets
    verbs:
      - get
      - list
      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/exec
+    verbs:
+      - create
+  - apiGroups:
+      - "batch"
+    resources:
+      - cronjobs
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "batch"
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
@ -93,6 +120,10 @@ spec:
                  key: 'POSTGRES_PASSWORD'
            - name: POSTGRESQL_HOST_PORT
              value: {{ tuple "postgresql" "internal" "postgresql" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+            - name: BACKUP_RESTORE_SCOPE
+              value: "postgresql"
+            - name: BACKUP_RESTORE_NAMESPACE_LIST
+              value: {{ .Values.conf.postgresql_backup_restore.enabled_namespaces | quote }}
          volumeMounts:
            - name: postgresql-utility-bin
              mountPath: /tmp/bootstrap.sh
@ -110,6 +141,18 @@ spec:
              mountPath: /usr/local/bin/postgresql-utility-rootwrap
              subPath: postgresql-utility-rootwrap
              readOnly: true
+            - name: postgresql-utility-bin
+              mountPath: /usr/local/bin/dbutils
+              subPath: dbutils
+              readOnly: true
+            - name: postgresql-utility-bin
+              mountPath: /tmp/pgutils.sh
+              subPath: pgutils.sh
+              readOnly: true
+            - name: postgresql-utility-bin
+              mountPath: /tmp/pg-ondemand-job.sh
+              subPath: pg-ondemand-job.sh
+              readOnly: true
            - name: postgresql-utility-sudoers
              mountPath: /etc/sudoers.d/utilscli-sudo
              subPath: utilscli-sudo
@ -126,7 +169,6 @@ spec:
              mountPath: /etc/postgresql/rootwrap.conf
              subPath: rootwrap.conf
              readOnly: true
-
      volumes:
        - name: postgresql-utility-sudoers
          configMap:
--- a/charts/postgresql-utility/values.yaml
+++ b/charts/postgresql-utility/values.yaml
@ -42,6 +42,17 @@ pod:
        postgresql_utility:
          allowPrivilegeEscalation: true
          readOnlyRootFilesystem: false
+    postgresql_ondemand:
+      pod:
+        runAsUser: 65534
+      container:
+        backup_perms:
+          runAsUser: 0
+          readOnlyRootFilesystem: true
+        postgresql_ondemand:
+          runAsUser: 65534
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
  dns_policy: "ClusterFirstWithHostNet"
  replicas:
    utility: 1
@ -79,12 +90,24 @@ pod:
        limits:
          memory: "1024Mi"
          cpu: "2000m"
+      postgresql_ondemand:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"

 secrets:
  postgresql:
    admin: postgresql-admin

 conf:
+  postgresql_backup_restore:
+    enabled_namespaces: "openstack"
+    secrets:
+      rgw_secret: postgresql-backup-user
+      conf_secret: postgresql-backup-restore
  features:
    utility: true
  postgresqlconf:
@ -96,6 +119,7 @@ conf:
      # Below are example command filters. access to postgresql cluster can be restricted by creating a user with less privilages
      psql: CommandFilter, psql, root
      kubectl: CommandFilter, kubectl, root
+      dbutils: CommandFilter, dbutils, nobody
  postgresqlrootwrap:
    DEFAULT:
      # Configuration for postgresql-rootwrap
@ -188,4 +212,4 @@ manifests:
  configmap_etc: true
  secret_etc: true
  secret_admin: true
-  deployment_utility: true
+  deployment_utility: true
--- a/images/postgresql-utility/Dockerfile.ubuntu_bionic
+++ b/images/postgresql-utility/Dockerfile.ubuntu_bionic
@ -8,27 +8,35 @@ LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc
      org.opencontainers.image.vendor='The Airship Authors' \
      org.opencontainers.image.licenses='Apache-2.0'

+ARG KUBE_VERSION=1.17.3
+
 RUN set -xe && \
    export DEBIAN_FRONTEND=noninteractive && \
    sed -i '/nobody/d' /etc/passwd && \
    echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd && \
    apt-get update && \
    apt-get install -y wget curl \
-    apt-transport-https ca-certificates gnupg \
-    bash \
-    moreutils \
-    rsyslog \
-    screen \
-    sudo \
-    postgresql-client \
-    postgresql-common \
-    python3.6 \
-    python3-pip && \
+        apt-transport-https ca-certificates gnupg \
+        bash \
+        moreutils \
+        rsyslog \
+        screen \
+        sudo \
+        postgresql-client \
+        postgresql-common \
+        python3.6 \
+        python3-pip && \
    pip3 install \
-      oslo.rootwrap==5.8.0 \
-      openstackclient \
-      python-swiftclient && \
+        oslo.rootwrap==5.8.0 \
+        openstackclient \
+        python-swiftclient && \
    apt-get clean -y && \
+    TMP_DIR=$(mktemp --directory) && \
+    cd ${TMP_DIR} && \
+    curl -sSL https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 && \
+    mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl && \
+    chmod +x /usr/bin/kubectl && \
+    rm -rf ${TMP_DIR} && \
    rm -rf /var/cache/debconf/* /var/lib/apt/lists/*

 RUN sed -i "/rootwrap_logger.setLevel/s/.*/#&/" /usr/local/lib/python3.6/dist-packages/oslo_rootwrap/wrapper.py \