From 3205c36c14c3fb9acde501a92d77e92926305798 Mon Sep 17 00:00:00 2001 From: Matt Pryor Date: Thu, 22 Sep 2022 11:23:14 +0100 Subject: [PATCH] Convert addon jobs to addon-provider resources --- charts/addon/.helmignore | 23 - charts/addon/Chart.yaml | 8 - charts/addon/templates/_helpers.tpl | 372 --------------- charts/addon/templates/_job-install.tpl | 160 ------- charts/addon/templates/_job-uninstall.tpl | 138 ------ charts/addon/templates/_secret.tpl | 59 --- charts/addon/values.yaml | 0 charts/cluster-addons/.helmignore | 23 - charts/cluster-addons/Chart.yaml | 11 - charts/cluster-addons/README.md | 399 ---------------- charts/cluster-addons/templates/_helpers.tpl | 250 ---------- .../templates/ccm-openstack.yaml | 43 -- .../templates/cert-manager.yaml | 37 -- .../templates/cloud-config.yaml | 101 ----- .../cluster-addons/templates/cni-calico.yaml | 29 -- .../cluster-addons/templates/cni-cilium.yaml | 44 -- .../cluster-addons/templates/csi-cinder.yaml | 50 -- .../templates/extra-addons.yaml | 26 -- .../templates/ingress-nginx.yaml | 13 - .../templates/kube-prometheus-stack.yaml | 31 -- .../templates/kubernetes-dashboard.yaml | 13 - .../cluster-addons/templates/loki-stack.yaml | 52 --- .../templates/mellanox-network-operator.yaml | 13 - .../templates/metrics-server.yaml | 21 - charts/cluster-addons/templates/nfd.yaml | 13 - .../templates/nvidia-gpu-operator.yaml | 13 - .../templates/prometheus-operator-crds.yaml | 22 - .../templates/purge-cloud-resources.yaml | 102 ----- .../templates/rbac/clusterrolebinding.yaml | 15 - .../cluster-addons/templates/rbac/role.yaml | 44 -- .../templates/rbac/rolebinding.yaml | 15 - .../templates/rbac/serviceaccount.yaml | 7 - charts/cluster-addons/values.yaml | 429 ------------------ charts/openstack-cluster/Chart.yaml | 7 - .../grafana-dashboards/loki-dashboard.json | 0 .../nvidia-dcgm-exporter-dashboard_rev2.json | 0 .../openstack-cluster/templates/_helpers.tpl | 50 +- .../templates/addons/_hook.tpl.yaml | 69 +++ .../templates/addons/cni/calico.yaml | 61 +++ .../templates/addons/cni/cilium.yaml | 40 ++ .../templates/addons/ingress-nginx.yaml | 29 ++ .../addons/kubernetes-dashboard.yaml | 36 ++ .../addons/mellanox-network-operator.yaml | 58 +++ .../templates/addons/metrics-server.yaml | 49 ++ .../monitoring/kube-prometheus-stack.yaml | 79 ++++ .../addons/monitoring/loki-stack.yaml | 103 +++++ .../templates/addons/nfd.yaml | 65 +++ .../templates/addons/nvidia-gpu-operator.yaml | 51 +++ .../templates/addons/openstack/ccm.yaml | 66 +++ .../addons/openstack/cloud-config.yaml | 64 +++ .../addons/openstack/csi-cinder.yaml | 116 +++++ .../templates/cluster-openstack.yaml | 2 +- .../openstack-cluster/templates/cluster.yaml | 2 +- .../control-plane/kubeadm-control-plane.yaml | 2 +- .../control-plane/machine-health-check.yaml | 2 +- .../node-group/kubeadm-config-template.yaml | 4 +- .../node-group/machine-deployment.yaml | 12 +- .../node-group/machine-health-check.yaml | 8 +- .../openstack-machine-template.yaml | 4 +- charts/openstack-cluster/values.yaml | 261 +++++++++-- utils/Dockerfile | 76 +--- utils/bin/helm-adopt | 65 +++ utils/bin/helm-delete | 37 -- utils/bin/helm-exists | 31 -- utils/bin/helm-move | 80 ++++ utils/bin/helm-upgrade | 54 --- utils/bin/make-chart | 66 --- 67 files changed, 1317 insertions(+), 2908 deletions(-) delete mode 100644 charts/addon/.helmignore delete mode 100644 charts/addon/Chart.yaml delete mode 100644 charts/addon/templates/_helpers.tpl delete mode 100644 charts/addon/templates/_job-install.tpl delete mode 100644 charts/addon/templates/_job-uninstall.tpl delete mode 100644 charts/addon/templates/_secret.tpl delete mode 100644 charts/addon/values.yaml delete mode 100644 charts/cluster-addons/.helmignore delete mode 100644 charts/cluster-addons/Chart.yaml delete mode 100644 charts/cluster-addons/README.md delete mode 100644 charts/cluster-addons/templates/_helpers.tpl delete mode 100644 charts/cluster-addons/templates/ccm-openstack.yaml delete mode 100644 charts/cluster-addons/templates/cert-manager.yaml delete mode 100644 charts/cluster-addons/templates/cloud-config.yaml delete mode 100644 charts/cluster-addons/templates/cni-calico.yaml delete mode 100644 charts/cluster-addons/templates/cni-cilium.yaml delete mode 100644 charts/cluster-addons/templates/csi-cinder.yaml delete mode 100644 charts/cluster-addons/templates/extra-addons.yaml delete mode 100644 charts/cluster-addons/templates/ingress-nginx.yaml delete mode 100644 charts/cluster-addons/templates/kube-prometheus-stack.yaml delete mode 100644 charts/cluster-addons/templates/kubernetes-dashboard.yaml delete mode 100644 charts/cluster-addons/templates/loki-stack.yaml delete mode 100644 charts/cluster-addons/templates/mellanox-network-operator.yaml delete mode 100644 charts/cluster-addons/templates/metrics-server.yaml delete mode 100644 charts/cluster-addons/templates/nfd.yaml delete mode 100644 charts/cluster-addons/templates/nvidia-gpu-operator.yaml delete mode 100644 charts/cluster-addons/templates/prometheus-operator-crds.yaml delete mode 100644 charts/cluster-addons/templates/purge-cloud-resources.yaml delete mode 100644 charts/cluster-addons/templates/rbac/clusterrolebinding.yaml delete mode 100644 charts/cluster-addons/templates/rbac/role.yaml delete mode 100644 charts/cluster-addons/templates/rbac/rolebinding.yaml delete mode 100644 charts/cluster-addons/templates/rbac/serviceaccount.yaml delete mode 100644 charts/cluster-addons/values.yaml rename charts/{cluster-addons => openstack-cluster}/grafana-dashboards/loki-dashboard.json (100%) rename charts/{cluster-addons => openstack-cluster}/grafana-dashboards/nvidia-dcgm-exporter-dashboard_rev2.json (100%) create mode 100644 charts/openstack-cluster/templates/addons/_hook.tpl.yaml create mode 100644 charts/openstack-cluster/templates/addons/cni/calico.yaml create mode 100644 charts/openstack-cluster/templates/addons/cni/cilium.yaml create mode 100644 charts/openstack-cluster/templates/addons/ingress-nginx.yaml create mode 100644 charts/openstack-cluster/templates/addons/kubernetes-dashboard.yaml create mode 100644 charts/openstack-cluster/templates/addons/mellanox-network-operator.yaml create mode 100644 charts/openstack-cluster/templates/addons/metrics-server.yaml create mode 100644 charts/openstack-cluster/templates/addons/monitoring/kube-prometheus-stack.yaml create mode 100644 charts/openstack-cluster/templates/addons/monitoring/loki-stack.yaml create mode 100644 charts/openstack-cluster/templates/addons/nfd.yaml create mode 100644 charts/openstack-cluster/templates/addons/nvidia-gpu-operator.yaml create mode 100644 charts/openstack-cluster/templates/addons/openstack/ccm.yaml create mode 100644 charts/openstack-cluster/templates/addons/openstack/cloud-config.yaml create mode 100644 charts/openstack-cluster/templates/addons/openstack/csi-cinder.yaml create mode 100755 utils/bin/helm-adopt delete mode 100755 utils/bin/helm-delete delete mode 100755 utils/bin/helm-exists create mode 100755 utils/bin/helm-move delete mode 100755 utils/bin/helm-upgrade delete mode 100755 utils/bin/make-chart diff --git a/charts/addon/.helmignore b/charts/addon/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/charts/addon/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/charts/addon/Chart.yaml b/charts/addon/Chart.yaml deleted file mode 100644 index fd638dc..0000000 --- a/charts/addon/Chart.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v2 -name: addon -description: > - Helm chart that provides templates for producing jobs that install addons onto a target cluster. - The target cluster can be local (using a service account) or remote (using a kubeconfig file). -type: library -version: 0.1.0 -appVersion: main diff --git a/charts/addon/templates/_helpers.tpl b/charts/addon/templates/_helpers.tpl deleted file mode 100644 index 07e8a87..0000000 --- a/charts/addon/templates/_helpers.tpl +++ /dev/null @@ -1,372 +0,0 @@ -{{- define "addon.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{- define "addon.fullname" -}} -{{- $ctx := index . 0 }} -{{- $componentName := index . 1 }} -{{- if $ctx.Values.fullnameOverride }} -{{- printf "%s-%s" $ctx.Values.fullnameOverride $componentName | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default $ctx.Chart.Name $ctx.Values.nameOverride }} -{{- if contains $name $ctx.Release.Name }} -{{- printf "%s-%s" $ctx.Release.Name $componentName | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s-%s" $ctx.Release.Name $name $componentName | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{- define "addon.job.name" -}} -{{- $ctx := index . 0 }} -{{- $componentName := index . 1 }} -{{- $operation := index . 2 }} -{{- $fullname := include "addon.fullname" (list $ctx $componentName) }} -{{- printf "%s-%s" $fullname $operation | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{- define "addon.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{- define "addon.selectorLabels" -}} -{{- $ctx := index . 0 -}} -{{- $componentName := index . 1 -}} -app.kubernetes.io/name: {{ include "addon.name" $ctx }} -app.kubernetes.io/instance: {{ $ctx.Release.Name }} -app.kubernetes.io/component: {{ $componentName }} -{{- end }} - -{{- define "addon.job.selectorLabels" -}} -{{- $ctx := index . 0 -}} -{{- $componentName := index . 1 -}} -{{- $operation := index . 2 -}} -{{ include "addon.selectorLabels" (list $ctx $componentName) }} -capi.stackhpc.com/operation: {{ $operation }} -{{- end }} - -{{- define "addon.commonLabels" -}} -helm.sh/chart: {{ include "addon.chart" . }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -{{- end }} - -{{- define "addon.labels" -}} -{{- $ctx := index . 0 -}} -{{ include "addon.commonLabels" $ctx }} -{{ include "addon.selectorLabels" . }} -{{- end }} - -{{- define "addon.job.labels" -}} -{{- $ctx := index . 0 -}} -{{ include "addon.commonLabels" $ctx }} -{{ include "addon.job.selectorLabels" . }} -capi.stackhpc.com/revision: {{ $ctx.Release.Revision | quote }} -{{- end }} - -{{/* -Template that merges two variables with the latter taking precedence and outputs the result as YAML. -Lists are merged by concatenating them rather than overwriting. -*/}} -{{- define "addon.mergeConcat" -}} -{{- $left := index . 0 }} -{{- if kindIs (kindOf list) $left }} -{{- index . 1 | default list | concat $left | toYaml }} -{{- else if kindIs (kindOf dict) $left }} -{{- $right := index . 1 | default dict }} -{{- if or $left $right }} -{{- range $key := concat (keys $left) (keys $right) | uniq }} -{{ $key }}: - {{- if and (hasKey $left $key) (hasKey $right $key) }} - {{- - include "addon.mergeConcat" (list (index $left $key) (index $right $key)) | - nindent 2 - }} - {{- else if hasKey $left $key }} - {{- index $left $key | toYaml | nindent 2 }} - {{- else }} - {{- index $right $key | toYaml | nindent 2 }} - {{- end }} -{{- end }} -{{- else }} -{} -{{- end }} -{{- else }} -{{- $right := index . 1 }} -{{- kindIs "invalid" $right | ternary $left $right | toYaml }} -{{- end }} -{{- end }} - -{{/* -Template for a Helm values file that consists of the given values merged with the -values obtained from rendering the valuesTemplate. -*/}} -{{- define "addon.helm.values" }} -{{- $ctx := index . 0 }} -{{- $config := index . 2 }} -{{- if $config.release.valuesTemplate }} -{{- $templateValues := tpl $config.release.valuesTemplate $ctx | fromYaml }} -{{- include "addon.mergeConcat" (list $config.release.values $templateValues) }} -{{- else }} -{{- toYaml $config.release.values }} -{{- end }} -{{- end }} - -{{/* -Template for a script that installs or upgrades a Helm release. - -Because Helm has poor support for CRDs, there is an option to apply CRD manifest URLs before -installing or upgrading the release. CRDs are installed using "kubectl create/replace" -rather than "kubectl apply" because CRDs with comprehensive schemas can easily become too -large for the last-applied-configuration annotation. - -There is also support for rolling back an interrupted install or upgrade before proceeding -by checking for the pending-[install,upgrade] status. -*/}} -{{- define "addon.helm.install" -}} -{{- $name := index . 0 }} -{{- $config := index . 1 }} -{{- - $chartRepo := required - "chart.repo is required for a Helm job" - $config.chart.repo -}} -{{- - $chartName := required - "chart.name is required for a Helm job" - $config.chart.name -}} -{{- - $chartVersion := required - "chart.version is required for a Helm job" - $config.chart.version -}} -{{- - $releaseNamespace := required - "release.namespace is required for a Helm job" - $config.release.namespace -}} -{{- if $config.crdManifests }} -crd_apply() { kubectl replace -f "$1" || kubectl create -f "$1"; } -{{- range $config.crdManifests }} -crd_apply {{ . }} -{{- end }} -{{- end }} -helm-upgrade {{ $name }} {{ $chartName }} \ - --atomic \ - --install \ - --namespace {{ $releaseNamespace }} \ - --create-namespace \ - --repo {{ $chartRepo }} \ - --version {{ $chartVersion }} \ - {{- if $config.crdManifests -}} - --skip-crds \ - {{- end }} - --values values.yaml \ - --wait \ - --wait-for-jobs \ - --timeout 24h \ - $HELM_EXTRA_ARGS -{{- end }} - -{{/* -Template for a script that deletes a Helm release. -*/}} -{{- define "addon.helm.delete" -}} -{{- $name := index . 0 }} -{{- $config := index . 1 }} -{{- - $releaseNamespace := required - "release.namespace is required for a Helm job" - $config.release.namespace -}} -helm-delete {{ $name }} \ - --namespace {{ $releaseNamespace }} \ - --wait \ - --timeout 24h -{{- range $config.crdManifests }} -kubectl delete -f {{ . }} -{{- end }} -{{- end }} - -{{/* -Template for a kustomization file for use with Kustomize. -*/}} -{{- define "addon.kustomize.kustomization" }} -{{- $ctx := index . 0 }} -{{- $config := index . 2 }} -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -{{- if $config.kustomizationTemplate }} -{{- $templateValues := tpl $config.kustomizationTemplate $ctx | fromYaml }} -{{ include "addon.mergeConcat" (list $config.kustomization $templateValues) }} -{{- else if $config.kustomization }} -{{ toYaml $config.kustomization }} -{{- else }} -{{- fail "One of kustomization or kustomizationTemplate is required for a Kustomize job" }} -{{- end }} -{{- end }} - -{{/* -Template for a script that installs or upgrades resources using Kustomize. - -Because kustomize has no release semantics, which we want, what we actually do is -convert the output of kustomize into an ephemeral Helm chart which is then installed -with no values. -*/}} -{{- define "addon.kustomize.install" }} -{{- $name := index . 0 }} -{{- $config := index . 1 }} -TMPDIR="$(mktemp -d)" -kustomize build . | make-chart {{ $name }} "$TMPDIR" -# Install the CRDs separately as Helm doesn't install updates -for crdfile in $(find "$TMPDIR/crds" -name '*.yaml'); do - kubectl create -f "$crdfile" || kubectl replace -f "$crdfile" -done -helm-upgrade {{ $name }} "$TMPDIR/chart" \ - --atomic \ - --install \ - --namespace kustomize-releases \ - --create-namespace \ - --skip-crds \ - --wait \ - --wait-for-jobs \ - --timeout 24h \ - $HELM_EXTRA_ARGS -{{- end }} - -{{/* -Template for a script that deletes resources using Kustomize. - -Because we are using Helm releases to manage resources templated by kustomize, -this just means deleting the Helm release. However we still want to run customize -in order to generate the CRDs that need deleting. -*/}} -{{- define "addon.kustomize.delete" }} -{{- $name := index . 0 }} -{{- $config := index . 1 }} -TMPDIR="$(mktemp -d)" -kustomize build . | make-chart {{ $name }} "$TMPDIR" -helm-delete {{ $name }} \ - --namespace kustomize-releases \ - --wait \ - --timeout 24h -for crdfile in $(find "$TMPDIR/crds" -name '*.yaml'); do - kubectl delete -f "$crdfile" -done -{{- end }} - -{{/* -Template that produces the default configuration. -*/}} -{{- define "addon.config.defaults" -}} -# Indicates whether the addon is enabled or not -enabled: true -# A list of other addons that this addon should wait for before installing -dependsOn: [] -# The weight to use for the uninstall hook -# This can be used to influence the order in which addons are deleted -uninstallHookWeight: 0 -image: - repository: ghcr.io/stackhpc/k8s-utils - tag: # Defaults to chart appVersion if not given - pullPolicy: IfNotPresent -imagePullSecrets: -kubeconfigSecret: - name: - key: value -serviceAccountName: -# One of helm, kustomize or custom -installType: custom -helm: - crdManifests: [] - chart: - repo: - name: - version: - release: - namespace: - # The template is rendered with the root context, then the result is merged into the dict - # Values from the template take precedence over the dict - values: {} - valuesTemplate: -kustomize: - # The template is rendered with the root context, then the result is merged into the dict - # Values from the template take precedence over the dict - kustomization: {} - kustomizationTemplate: -custom: - # Scripts are treated as templates during rendering - install: - delete: -extraVolumes: [] -extraFiles: {} -# The hook scripts are treated as templates during the rendering -hooks: - preInstall: - postInstall: - preDelete: - postDelete: -# Extra containers to run as init containers -# These should include environment variables, volume mounts etc. if they need -# to target a remote cluster using kubeconfigSecret -extraInitContainers: [] -# Indicates whether a pre-delete hook should be generated for the addon -generatePreDeleteHook: true -backoffLimit: 1000 -activeDeadlineSeconds: 3600 -podSecurityContext: - runAsNonRoot: true -securityContext: - allowPrivilegeEscalation: false -resources: {} -hostNetwork: false -tolerations: [] -nodeSelector: {} -affinity: {} -{{- end }} - -{{/* -Template that produces a config secret, an install job and a hooks for the specified addon. - -If the addon is enabled, an install job is produced as part of the main release and a pre-delete -hook is also produced. - -If the addon is disabled, then we check if the config secret exists for the addon. If it does, a -pre-upgrade hook is produced to uninstall the addon. -*/}} -{{- define "addon.job.fromConfig" -}} -{{- $ctx := index . 0 }} -{{- $name := index . 1 }} -{{- $overrides := index . 2 }} -{{- $defaults := include "addon.config.defaults" $ctx | fromYaml }} -{{- $config := include "addon.mergeConcat" (list $defaults $overrides) | fromYaml }} -{{- if $config.enabled }} -{{- include "addon.config.secret" (list $ctx $name $config) }} ---- -{{- include "addon.job.install" (list $ctx $name $config) }} -{{- if $config.generatePreDeleteHook }} ---- -{{- include "addon.job.uninstall" (list $ctx $name "pre-delete" $config) }} -{{- end }} -{{- else if $ctx.Release.IsUpgrade }} -{{- $secretName := include "addon.fullname" (list $ctx $name) | printf "%s-config" }} -{{- if lookup "v1" "Secret" $ctx.Release.Namespace $secretName }} -{{- include "addon.job.uninstall" (list $ctx $name "pre-upgrade" $config) }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Template that produces a config secret, an install job and a delete hook -for the configuration produced by the specified template. -*/}} -{{- define "addon.job" -}} -{{- $ctx := index . 0 }} -{{- $name := index . 1 }} -{{- $configTemplate := index . 2 }} -{{- $config := include $configTemplate $ctx | fromYaml }} -{{- include "addon.job.fromConfig" (list $ctx $name $config) }} -{{- end }} diff --git a/charts/addon/templates/_job-install.tpl b/charts/addon/templates/_job-install.tpl deleted file mode 100644 index 364f648..0000000 --- a/charts/addon/templates/_job-install.tpl +++ /dev/null @@ -1,160 +0,0 @@ -{{/* - In order to only run jobs when something has changed, we include a fragment of - the checksum of the job spec in the job name. The pod template contains the - checksum of the configuration as an annotation, so that the job spec changes when - the configuration does. - - This guarantees we get new jobs only when there is a change to make. Even if there is - not a new job, the job labels are updated to include the current revision so that we - can order the jobs in time. - - The job spec is immutable, which can cause issues with updates. To mitigate this, we - use the spec from the existing job when a job exists with the same name (and hence - the same checksum). -*/}} -{{- define "addon.job.install.spec" -}} -{{- $ctx := index . 0 -}} -{{- $name := index . 1 -}} -{{- $config := index . 2 -}} -backoffLimit: {{ $config.backoffLimit }} -activeDeadlineSeconds: {{ $config.activeDeadlineSeconds }} -template: - metadata: - labels: {{ include "addon.job.selectorLabels" (list $ctx $name "install") | nindent 6 }} - annotations: - capi.stackhpc.com/config-checksum: {{ include "addon.config.secret" . | sha256sum }} - spec: - {{- with $config.imagePullSecrets }} - imagePullSecrets: {{ toYaml . | nindent 6 }} - {{- end }} - securityContext: {{ toYaml $config.podSecurityContext | nindent 6 }} - restartPolicy: OnFailure - serviceAccountName: {{ tpl $config.serviceAccountName $ctx }} - initContainers: - - name: wait-for-api - image: {{ printf "%s:%s" $config.image.repository (default $ctx.Chart.AppVersion $config.image.tag) }} - imagePullPolicy: {{ $config.image.pullPolicy }} - securityContext: {{ toYaml $config.securityContext | nindent 10 }} - args: - - /bin/bash - - -c - - | - set -x - until kubectl api-resources >/dev/null 2>&1; do - sleep 5 - done - {{- if $config.kubeconfigSecret.name }} - env: - - name: KUBECONFIG - value: /config/kubeconfig - {{- end }} - resources: {{ toYaml $config.resources | nindent 10 }} - volumeMounts: - - name: config - mountPath: /config - readOnly: true - {{- if $config.dependsOn }} - - name: wait-for-dependencies - image: {{ printf "%s:%s" $config.image.repository (default $ctx.Chart.AppVersion $config.image.tag) }} - imagePullPolicy: {{ $config.image.pullPolicy }} - securityContext: {{ toYaml $config.securityContext | nindent 10 }} - args: - - /bin/bash - - -c - - | - set -ex - {{- range $dep := $config.dependsOn }} - {{- $labels := include "addon.job.selectorLabels" (list $ctx $dep "install") | fromYaml }} - {{- range $i, $label := (keys $labels | sortAlpha) -}} - {{- if $i }} - LABELS="$LABELS,{{ $label }}={{ index $labels $label }}" - {{- else }} - LABELS="{{ $label }}={{ index $labels $label }}" - {{- end }} - {{- end }} - kubectl wait --for=condition=Complete job -n {{ $ctx.Release.Namespace }} -l "$LABELS" --all --timeout=-1s - {{- end }} - resources: {{ toYaml $config.resources | nindent 10 }} - {{- end }} - {{- range $config.extraInitContainers }} - - {{ toYaml . | nindent 8 }} - {{- end }} - containers: - - name: install - image: {{ printf "%s:%s" $config.image.repository (default $ctx.Chart.AppVersion $config.image.tag) }} - imagePullPolicy: {{ $config.image.pullPolicy }} - securityContext: {{ toYaml $config.securityContext | nindent 10 }} - args: - - /bin/bash - - -c - - | - set -exo pipefail - {{- if $config.hooks.preInstall }} - source ./hook-preinstall.sh - {{- end }} - source ./install.sh - {{- if $config.hooks.postInstall }} - source ./hook-postinstall.sh - {{- end }} - {{- if $config.kubeconfigSecret.name }} - env: - - name: KUBECONFIG - value: /config/kubeconfig - {{- end }} - # Set the working directory to the directory containing the config - workingDir: /config - resources: {{ toYaml $config.resources | nindent 10 }} - volumeMounts: - - name: config - mountPath: /config - readOnly: true - hostNetwork: {{ $config.hostNetwork }} - {{- with $config.nodeSelector }} - nodeSelector: {{ toYaml . | nindent 6 }} - {{- end }} - {{- with $config.affinity }} - affinity: {{ toYaml . | nindent 6 }} - {{- end }} - {{- with $config.tolerations }} - tolerations: {{ toYaml . | nindent 6 }} - {{- end }} - volumes: - - name: config - projected: - sources: - - secret: - name: {{ include "addon.fullname" (list $ctx $name) }}-config - {{- if $config.kubeconfigSecret.name }} - - secret: - name: {{ tpl $config.kubeconfigSecret.name $ctx }} - items: - - key: {{ $config.kubeconfigSecret.key }} - path: kubeconfig - {{- end }} - {{- range $config.extraVolumes }} - - {{ toYaml . | nindent 14 }} - {{- end }} -{{- end }} - -{{- define "addon.job.install" -}} -{{- $ctx := index . 0 }} -{{- $name := index . 1 }} -apiVersion: batch/v1 -kind: Job -metadata: - {{- $checksum := include "addon.job.install.spec" . | sha256sum }} - {{- - $jobName := printf "%s-%s" - (include "addon.job.name" (list $ctx $name "install") | trunc 57 | trimSuffix "-") - (trunc 5 $checksum) - }} - name: {{ $jobName }} - labels: {{ include "addon.job.labels" (list $ctx $name "install") | nindent 4 }} -spec: - {{- $existingJob := lookup "batch/v1" "Job" $ctx.Release.Namespace $jobName }} - {{- if $existingJob }} - {{- toYaml $existingJob.spec | nindent 2 }} - {{- else }} - {{- include "addon.job.install.spec" . | nindent 2 }} - {{- end }} -{{- end }} diff --git a/charts/addon/templates/_job-uninstall.tpl b/charts/addon/templates/_job-uninstall.tpl deleted file mode 100644 index 711d67a..0000000 --- a/charts/addon/templates/_job-uninstall.tpl +++ /dev/null @@ -1,138 +0,0 @@ -{{- define "addon.job.uninstall" -}} -{{- $ctx := index . 0 }} -{{- $name := index . 1 }} -{{- $hook := index . 2 }} -{{- $config := index . 3 }} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "addon.job.name" (list $ctx $name "uninstall") }} - labels: {{ include "addon.job.labels" (list $ctx $name "uninstall") | nindent 4 }} - annotations: - helm.sh/hook: {{ $hook }} - helm.sh/hook-weight: {{ $config.uninstallHookWeight | quote }} - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - backoffLimit: {{ $config.backoffLimit }} - activeDeadlineSeconds: {{ $config.activeDeadlineSeconds }} - template: - metadata: - labels: {{ include "addon.job.selectorLabels" (list $ctx $name "uninstall") | nindent 8 }} - spec: - {{- with $config.imagePullSecrets }} - imagePullSecrets: {{ toYaml . | nindent 8 }} - {{- end }} - securityContext: {{ toYaml $config.podSecurityContext | nindent 8 }} - restartPolicy: OnFailure - serviceAccountName: {{ tpl $config.serviceAccountName $ctx }} - # Use init containers to do two things before uninstalling - # - #  1. Suspend any running install jobs for the addon - #  2. Install the kubeconfig file from the secret if required - # - # We don't use a regular volume for (2) because we need the hook not to block in the - # case where the secret is not available - initContainers: - - name: suspend-install-jobs - image: {{ printf "%s:%s" $config.image.repository (default $ctx.Chart.AppVersion $config.image.tag) }} - imagePullPolicy: {{ $config.image.pullPolicy }} - securityContext: {{ toYaml $config.securityContext | nindent 12 }} - args: - - /bin/bash - - -c - - | - set -ex - {{- $labels := include "addon.job.selectorLabels" (list $ctx $name "install") | fromYaml }} - {{- range $i, $label := keys $labels -}} - {{- if $i }} - LABELS="$LABELS,{{ $label }}={{ index $labels $label }}" - {{- else }} - LABELS="{{ $label }}={{ index $labels $label }}" - {{- end }} - {{- end }} - for job in $(kubectl get job -n {{ $ctx.Release.Namespace }} -l "$LABELS" -o name); do - kubectl patch $job -n {{ $ctx.Release.Namespace }} -p '{"spec":{"suspend":true}}' - done - resources: {{ toYaml $config.resources | nindent 12 }} - {{- if $config.kubeconfigSecret.name }} - - name: install-kubeconfig - image: {{ printf "%s:%s" $config.image.repository (default $ctx.Chart.AppVersion $config.image.tag) }} - imagePullPolicy: {{ $config.image.pullPolicy }} - securityContext: {{ toYaml $config.securityContext | nindent 12 }} - args: - - /bin/bash - - -c - - | - set -ex - kubectl_get_secret() { - kubectl get secret {{ tpl $config.kubeconfigSecret.name $ctx }} -n {{ $ctx.Release.Namespace }} "$@" - } - if kubectl_get_secret; then - {{- $template := printf "{{ index .data \"%s\" | base64decode }}" $config.kubeconfigSecret.key }} - kubectl_get_secret -o go-template='{{ $template }}' > /config/auth/kubeconfig - fi - resources: {{ toYaml $config.resources | nindent 12 }} - volumeMounts: - - name: kubeconfig - mountPath: /config/auth - {{- end }} - containers: - - name: uninstall - image: {{ printf "%s:%s" $config.image.repository (default $ctx.Chart.AppVersion $config.image.tag) }} - imagePullPolicy: {{ $config.image.pullPolicy }} - securityContext: {{ toYaml $config.securityContext | nindent 12 }} - # We can only make a best effort to delete the addon as we don't want the hook to block - # So we bail without an error if the kubeconfig doesn't exist or the API is not reachable - # and we allow the scripts to fail without preventing execution of the following scripts - args: - - /bin/bash - - -c - - | - {{- if $config.kubeconfigSecret.name }} - test -f "$KUBECONFIG" || exit 0 - {{- end }} - kubectl version || exit 0 - {{- if $config.hooks.preDelete }} - source ./hook-predelete.sh || true - {{- end }} - source ./delete.sh || true - {{- if $config.hooks.postDelete }} - source ./hook-postdelete.sh || true - {{- end }} - {{- if $config.kubeconfigSecret.name }} - env: - - name: KUBECONFIG - value: /config/auth/kubeconfig - {{- end }} - # Set the working directory to the directory containing the config - workingDir: /config - resources: {{ toYaml $config.resources | nindent 12 }} - volumeMounts: - - name: config - mountPath: /config - readOnly: true - - name: kubeconfig - mountPath: /config/auth - readOnly: true - hostNetwork: {{ $config.hostNetwork }} - {{- with $config.nodeSelector }} - nodeSelector: {{ toYaml . | nindent 8 }} - {{- end }} - {{- with $config.affinity }} - affinity: {{ toYaml . | nindent 8 }} - {{- end }} - {{- with $config.tolerations }} - tolerations: {{ toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: config - projected: - sources: - - secret: - name: {{ include "addon.fullname" (list $ctx $name) }}-config - {{- range $config.extraVolumes }} - - {{ toYaml . | nindent 16 }} - {{- end }} - - name: kubeconfig - emptyDir: {} -{{- end }} diff --git a/charts/addon/templates/_secret.tpl b/charts/addon/templates/_secret.tpl deleted file mode 100644 index 6bc240b..0000000 --- a/charts/addon/templates/_secret.tpl +++ /dev/null @@ -1,59 +0,0 @@ -{{- define "addon.config.secret" -}} -{{- $ctx := index . 0 }} -{{- $name := index . 1 }} -{{- $config := index . 2 }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "addon.fullname" (list $ctx $name) }}-config - labels: {{ include "addon.labels" (list $ctx $name) | nindent 4 }} -stringData: - {{- range $filename, $content := $config.extraFiles }} - {{ $filename }}: | - {{- $content | nindent 4 }} - {{- end }} - {{- with $config.hooks.preInstall }} - hook-preinstall.sh: | - {{- tpl . $ctx | nindent 4 }} - {{- end }} - {{- with $config.hooks.postInstall }} - hook-postinstall.sh: | - {{- tpl . $ctx | nindent 4 }} - {{- end }} - {{- with $config.hooks.preDelete }} - hook-predelete.sh: | - {{- tpl . $ctx | nindent 4 }} - {{- end }} - {{- with $config.hooks.postDelete }} - hook-postdelete.sh: | - {{- tpl . $ctx | nindent 4 }} - {{- end }} - {{- if eq $config.installType "helm" }} - values.yaml: | - {{- include "addon.helm.values" (list $ctx $name $config.helm) | nindent 4 }} - install.sh: | - {{- include "addon.helm.install" (list $name $config.helm) | nindent 4 }} - delete.sh: | - {{- include "addon.helm.delete" (list $name $config.helm) | nindent 4 }} - {{- else if eq $config.installType "kustomize" }} - kustomization.yaml: | - {{- include "addon.kustomize.kustomization" (list $ctx $name $config.kustomize) | nindent 4 }} - install.sh: | - {{- include "addon.kustomize.install" (list $name $config.kustomize) | nindent 4 }} - delete.sh: | - {{- include "addon.kustomize.delete" (list $name $config.kustomize) | nindent 4 }} - {{- else if eq $config.installType "custom" }} - install.sh: | - {{- - tpl (required "custom.install is required for a custom job" $config.custom.install) $ctx | - nindent 4 - }} - delete.sh: | - {{- - tpl (required "custom.delete is required for a custom job" $config.custom.delete) $ctx | - nindent 4 - }} - {{- else }} - {{- fail (printf "Unrecognised install type '%s'" $config.installType) }} - {{- end }} -{{- end }} diff --git a/charts/addon/values.yaml b/charts/addon/values.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/charts/cluster-addons/.helmignore b/charts/cluster-addons/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/charts/cluster-addons/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/charts/cluster-addons/Chart.yaml b/charts/cluster-addons/Chart.yaml deleted file mode 100644 index c5a4735..0000000 --- a/charts/cluster-addons/Chart.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v2 -name: cluster-addons -description: Helm chart that deploys cluster addons for a CAPI cluster. -type: application -version: 0.1.0 -appVersion: main - -dependencies: - - name: addon - version: ">=0-0" - repository: file://../addon diff --git a/charts/cluster-addons/README.md b/charts/cluster-addons/README.md deleted file mode 100644 index 0c0db5f..0000000 --- a/charts/cluster-addons/README.md +++ /dev/null @@ -1,399 +0,0 @@ -# cluster-addons chart - -This [Helm chart](https://helm.sh/) manages the deployment of addons for a -[Kubernetes](https://kubernetes.io) cluster. It is primarily intended to be used with -the cluster management charts from this repository, e.g. -[openstack-cluster](../openstack-cluster), but should work for any Kubernetes cluster. - -## Contents - -- [How does it work?](#how-does-it-work) -- [Targetting a remote cluster](#targetting-a-remote-cluster) -- [Container Network Interface (CNI) plugins](#container-network-interface-cni-plugins) -- [OpenStack integrations](#openstack-integrations) - - [OpenStack credentials](#openstack-credentials) -- [cert-manager](#cert-manager) -- [Ingress controllers](#ingress-controllers) -- [Extra addons](#extra-addons) - -## How does it work? - -The addons are deployed by launching -[Kubernetes jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/), each of -which is responsible for installing or updating a single addon. These jobs can -either install the addons into the local cluster using a service account or a remote cluster -using a `kubeconfig` file in a pre-existing secret. By default, the local cluster is the -target. - -The jobs use the [utils image](../../utils) from this repository, which bundles some -useful tools like [jq](https://stedolan.github.io/jq/), -[kubectl](https://kubernetes.io/docs/reference/kubectl/overview/), -[kustomize](https://kustomize.io/) and [helm](https://helm.sh). - -When targetting the local cluster, the service account used to run the jobs must have -enough permissions to create all the objects that the addon will create. In practice, -this means that the service account will usually require the `cluster-admin` cluster role -for two reasons: - - 1. This chart provides a mechanism to specify custom addons, and there is no way to - know in advance what resources those custom addons may need to manage. - 1. This may even include instances of a - [CRD](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/) - that is installed by another addon. - 1. Several addons need to create - [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) resources, - and Kubernetes requires that the account creating RBAC resources has at least the - permissions that it is attempting to apply to another account. - -There are two patterns used in this chart for managing addons: - - 1. Manifests are pulled from a URL and run through `kustomize` before being applied - using `kubectl`. The manifests are **not** present in this repository. In this case, - the URL and kustomize configuration can be changed using the Helm values if required, - e.g. to change images from Docker Hub to another repository or to point to an - internal source if an air-gapped installation is required. - 1. Using a Helm chart. The chart to use is configured using Helm values rather - than Helm dependencies, which allows full control via configuration over which - repository is used (e.g. a mirror for an air-gapped installation) and which version - is installed. The Helm values for the addon are also exposed, and can be customised, - via the values for this chart. This chart sets sensible defaults. - -This chart also allows custom addons to be managed using the Helm values, either by -specifying manifest content inline, or by specifying a Helm chart to install with the -corresponding values. - -## Targetting a remote cluster - -By default, the jobs that install the addons target the local cluster using a service account. - -It is also possible to target a remote cluster, using a `kubeconfig` file. This must first -be uploaded to the cluster as a secret: - -```sh -kubectl create secret generic target-kubeconfig --from-file=kubeconfig=$PWD/kubeconfig -``` - -Then you can tell the addons to use that `kubeconfig` file using the Helm values: - -```yaml -kubeconfigSecret: - name: target-kubeconfig - key: kubeconfig -``` - -## Container Network Interface (CNI) plugins - -This chart can install either [Calico](https://docs.projectcalico.org/about/about-calico) or -[Cilium](https://cilium.io/) as a -[CNI plugin](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) -to provide the pod networking in a Kubernetes cluster. By default, the Calico CNI will be -installed. - -To switch the CNI to Cilium, use the following in your Helm values: - -```yaml -cni: - type: cilium -``` - -And to disable the installation of a CNI completely: - -```yaml -cni: - enabled: false -``` - -Additional configuration options are available for each - see [values.yaml](./values.yaml). - -## OpenStack integrations - -Kubernetes allows cloud providers to provide various plugins to integrate with the -underlying infrastructure, for example -[Cloud Controller Managers (CCMs)](https://kubernetes.io/docs/concepts/architecture/cloud-controller/), -[Container Storage Interface (CSI) implementations](https://kubernetes-csi.github.io/docs/) -and [authenticating webhooks](https://kubernetes.io/docs/reference/access-authn-authz/webhook/). - -This chart is able to deploy the CCM and the Cinder CSI plugin from the -[Kubernetes OpenStack cloud provider](https://github.com/kubernetes/cloud-provider-openstack), -which allows your Kubernetes cluster to integrate with the OpenStack cloud on which it is deployed. -This enables features like automatic labelling of nodes with OpenStack information (e.g. server ID -and flavor), automatic configuration of hostnames and IP addresses, managed load balancers for -services and dynamic provisioning of RWO volumes. - -By default, the OpenStack integrations are not enabled. To enable OpenStack integrations on the -target cluster, use the following in your Helm values: - -```yaml -openstack: - enabled: true -``` - -To configure options for the `[Networking]`, `[LoadBalancer]`, `[BlockStorage]` and `[Metadata]` -sections of the cloud-config file, you can use Helm values, e.g.: - -```yaml -openstack: - cloudConfig: - Networking: - public-network-name: public-internet - LoadBalancer: - lb-method: LEAST_CONNECTIONS - create-monitor: true - BlockStorage: - ignore-volume-az: true - Metadata: - search-order: metadataService -``` - -The `[Globals]` section is populated using the given `clouds.yaml` (see "OpenStack credentials" below). - -For the available options, consult the documentation for the -[CCM](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md#config-openstack-cloud-controller-manager) -and the -[Cinder CSI plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md#block-storage). - -Additional configuration options are available for the OpenStack integrations - see -[values.yaml](./values.yaml) for more details. - -### OpenStack credentials - -OpenStack credentials are required for the Kubernetes OpenStack integrations to query and -manage OpenStack resources on behalf of the cluster. The recommended way to do this is using an -[Application Credential](https://docs.openstack.org/keystone/latest/user/application_credentials.html) -to avoid your password being in stored on the cluster. Application credentials are project-scoped, -and ideally you should use a separate application credential for each cluster in a project. - -When an application credential is created in Horizon, the corresponding `clouds.yaml` file can be -downloaded, and should look something like this: - -```yaml -clouds: - openstack: - auth: - auth_url: https://my.cloud:5000 - application_credential_id: "" - application_credential_secret: "" - region_name: "RegionOne" - interface: "public" - identity_api_version: 3 - auth_type: "v3applicationcredential" -``` - -The credentials are provided to this Helm chart by putting them into a secret: - -```sh -kubectl create secret generic my-cloud-credential --from-file=clouds.yaml=$PWD/clouds.yaml -``` - -That secret can then be configured in the Helm values: - -```yaml -openstack: - cloudCredentialsSecretName: my-cloud-credential -``` - -The secret can also contain a certificate file that is used to validate the SSL certificate from -the target cloud: - -```sh -kubectl create secret generic my-cloud-credential \ - --from-file=clouds.yaml=$PWD/clouds.yaml \ - --from-file=cacert=$PWD/ca.crt -``` - -Alternatively, certificate verification can be disabled in the `clouds.yaml`: - -```yaml -clouds: - openstack: - auth: - auth_url: https://my.cloud:5000 - application_credential_id: "" - application_credential_secret: "" - region_name: "RegionOne" - interface: "public" - identity_api_version: 3 - auth_type: "v3applicationcredential" - verify: false -``` - -## cert-manager - -This chart is able to install [cert-manager](https://cert-manager.io/) on the target cluster. -cert-manager provides custom resources for managing X509 certificates and certificate -issuers as native Kuberbetes resources, simplifying the issuance and renewal of X509 -certificates (e.g. for TLS for web services). It is able to automatically negotiate -certificates from services such as [Let's Encrypt](https://letsencrypt.org/) by fulfilling -the required challenges, and can -[automatically issue certificates](https://cert-manager.io/docs/usage/ingress/) for -[Ingress resources](https://kubernetes.io/docs/concepts/services-networking/ingress/) -using annotations. - -cert-manager is disabled by default. To enable it, use the following Helm values: - -```yaml -certManager: - enabled: true -``` - -By default, the installation includes a cluster issuer called `letsencrypt-http01` that -targets [Let's Encrypt](https://letsencrypt.org/) for certificate issuing. - -Additional configuration options are available for cert-manager - see -[values.yaml](./values.yaml). - -## Ingress controllers - -Running an -[Ingress Controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/) -on your Kubernetes cluster enables the use of -[Ingress resource](https://kubernetes.io/docs/concepts/services-networking/ingress/) -to manage HTTP(S) traffic flowing in and out of the cluster. This allows your web applications -to take advantage of load-balancing, name-based virtual hosting, path-based routing and -TLS termination using the same declarative approach as other Kubernetes resources. -When combined with a cert-manager issuer (see above) this provides an almost frictionless way -to secure your web services. - -It is possible to install multiple Ingress Controllers and select the preferred one for a -particular Ingress resource using -[Ingress Classes](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class). - -This chart can install the [Nginx Ingress Controller](https://kubernetes.github.io/ingress-nginx/) -onto the target cluster. - -The Nginx Ingress Controller is disabled by default. To enable it, use the following Helm values: - -```yaml -ingress: - enabled: true -``` - -## Extra addons - -This chart is able to manage the application of additional user-specified addons to the target -cluster. These can use Helm, Kustomize or a custom script to install and uninstall the addon, -and can even use a custom image containing specialist tools if required. - -Each addon should have the form (not all options are required at all times): - -```yaml -# The name of any components that this addon depends on being installed before it can be installed -# This can be the name of an addon, including the name of other extra addons -# It can also be either "ingress" or "storage" to wait for storage providers and ingress controllers -# respectively, regardless of the explicit implementation -dependsOn: [] -# One of helm, kustomize or custom -installType: custom -# Options for a Helm addon -helm: - # List of URLs of manifests containing CRDs - # Helm's handling of CRDs is not great - this helps if CRDs require updates - crdManifests: [] - # The information for the Helm chart - chart: - # The URL of the chart repository - repo: - # The name of the chart - name: - # The version of the chart to use - version: - # Information about the Helm release - release: - # The namespace for the release on the target cluster - namespace: - # The values for the release - # These can come from a dict or a template - # The template is rendered with the root context, then the result is merged into the dict - # Values from the template take precedence over the dict - values: {} - valuesTemplate: -# Options for a kustomize addon -kustomize: - # The kustomize configuration - # This can come from a dict or a template - # The template is rendered with the root context, then the result is merged into the dict - # Values from the template take precedence over the dict - kustomization: {} - kustomizationTemplate: -# Options for a custom addon -custom: - # Script that installs the addon - # It is treated as a template, and rendered with the root context - install: - # Script that deletes the addon - # It is also treated as a template and rendered with the root context - delete: -# A list of extra sources to be added to the projected volume used for configuration -# The secrets and configmaps must already exist in the namespace -# https://kubernetes.io/docs/concepts/storage/projected-volumes/ -extraVolumes: [] -# A map of filename -> content of additional files to include in the config directory -extraFiles: {} -# Hook scripts that execute at certain times in the addon's lifecycle -# Hook scripts are treated as templates during rendering, and are rendered with the root context -hooks: - # Executed before the addon is installed or upgraded - preInstall: - # Executed after the addon is installed or upgraded - postInstall: - # Executed before the addon is deleted - preDelete: - # Executed after the addon is deleted - postDelete: -# Details of a custom image to use, if required -image: -  # The repository of the image - repository: - # The tag to use from the repository - tag: -``` - -For example, the following extra addon will install a couple of additional manifests -into the cluster using Kustomize: - -```yaml -extraAddons: - custom-manifests: - installType: kustomize - kustomize: - kustomization: - resources: - - ./storageclass-standard.yaml - - ./pod-reader.yaml - extraFiles: - storageclass-standard.yaml: | - apiVersion: storage.k8s.io/v1 - kind: StorageClass - metadata: - name: standard - provisioner: my-storage-provisioner - pod-reader.yaml: | - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: pod-reader - rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list"] -``` - -Or to deploy a custom Helm release as part of the addon installation: - -```yaml -extraAddons: - my-wordpress: - installType: helm - helm: - chart: - repo: https://charts.bitnami.com/bitnami - name: wordpress - version: 12.1.6 - release: - namespace: wordpress - name: my-wordpress - values: - wordpressUsername: jbloggs - wordpressPassword: supersecretpassword - wordpressBlogName: JBloggs Awesome Blog! -``` diff --git a/charts/cluster-addons/templates/_helpers.tpl b/charts/cluster-addons/templates/_helpers.tpl deleted file mode 100644 index fb46a9d..0000000 --- a/charts/cluster-addons/templates/_helpers.tpl +++ /dev/null @@ -1,250 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "cluster-addons.name" -}} -{{- .Chart.Name | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -*/}} -{{- define "cluster-addons.fullname" -}} -{{- if contains .Chart.Name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "cluster-addons.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Selector labels for a chart-level resource. -*/}} -{{- define "cluster-addons.selectorLabels" -}} -app.kubernetes.io/name: {{ include "cluster-addons.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Labels for a chart-level resource. -*/}} -{{- define "cluster-addons.labels" -}} -helm.sh/chart: {{ include "cluster-addons.chart" . }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -{{ include "cluster-addons.selectorLabels" . }} -{{- end }} - -{{/* -Renders the default job configuration. -*/}} -{{- define "cluster-addons.job.defaults" -}} -{{- $ctx := index . 0 }} -{{- $name := index . 1 }} -{{- with $ctx.Values.jobDefaults }} -{{- toYaml . }} -{{- end }} -{{- if $ctx.Values.kubeconfigSecret.name }} -kubeconfigSecret: - name: {{ tpl $ctx.Values.kubeconfigSecret.name $ctx }} - {{- with $ctx.Values.kubeconfigSecret.key }} - key: {{ . }} - {{- end }} -{{- end }} -serviceAccountName: {{ tpl $ctx.Values.serviceAccount.name $ctx }} -enabled: {{ include "cluster-addons.enabled" . }} -dependsOn: {{ - (include "cluster-addons.dependsOn.enabled" . | fromYaml).value | - default list | - toYaml | - nindent 2 -}} -uninstallHookWeight: {{ include "cluster-addons.uninstallHookWeight" . }} -{{- if and $ctx.Values.clusterApi (not (has $name $ctx.Values.categories.bootstrap)) }} -extraInitContainers: - - name: wait-for-capi-cluster - image: {{ - printf "%s:%s" - $ctx.Values.jobDefaults.image.repository - (default $ctx.Chart.AppVersion $ctx.Values.jobDefaults.image.tag) - }} - imagePullPolicy: {{ $ctx.Values.jobDefaults.image.pullPolicy }} - securityContext: {{ toYaml $ctx.Values.jobDefaults.securityContext | nindent 6 }} - args: - - kubectl - - wait - - --for=condition=Ready - - clusters.cluster.x-k8s.io - - {{ tpl $ctx.Values.clusterName $ctx }} - - --namespace - - {{ $ctx.Release.Namespace }} - - --timeout - - "-1s" - resources: {{ toYaml $ctx.Values.jobDefaults.resources | nindent 6 }} -{{- end }} -# If the addons are deployed as part of a Cluster API cluster, suppress the pre-delete hooks -# If the cluster no longer exists, then neither do the addons! -generatePreDeleteHook: {{ not $ctx.Values.clusterApi | toYaml }} -{{- end }} - -{{/* -Determines if an addon is enabled given the name. -*/}} -{{- define "cluster-addons.enabled" -}} -{{- $ctx := index . 0 -}} -{{- $name := index . 1 -}} -{{- if eq $name "ccm-openstack" -}} -{{- and $ctx.Values.openstack.enabled $ctx.Values.openstack.ccm.enabled | toYaml -}} -{{- else if eq $name "cert-manager" -}} -{{- $ctx.Values.certManager.enabled | toYaml -}} -{{- else if eq $name "cloud-config" -}} -{{- $ctx.Values.openstack.enabled | toYaml -}} -{{- else if eq $name "cni-calico" -}} -{{- and $ctx.Values.cni.enabled (eq $ctx.Values.cni.type "calico") | toYaml -}} -{{- else if eq $name "cni-cilium" -}} -{{- and $ctx.Values.cni.enabled (eq $ctx.Values.cni.type "cilium") | toYaml -}} -{{- else if eq $name "csi-cinder" -}} -{{- and $ctx.Values.openstack.enabled $ctx.Values.openstack.csiCinder.enabled | toYaml -}} -{{- else if eq $name "ingress-nginx" -}} -{{- and $ctx.Values.ingress.enabled $ctx.Values.ingress.nginx.enabled | toYaml -}} -{{- else if eq $name "kubernetes-dashboard" -}} -{{- $ctx.Values.kubernetesDashboard.enabled | toYaml -}} -{{- else if eq $name "metrics-server" -}} -{{- $ctx.Values.metricsServer.enabled | toYaml -}} -{{- else if eq $name "prometheus-operator-crds" -}} -{{- $ctx.Values.monitoring.enabled | toYaml -}} -{{- else if eq $name "kube-prometheus-stack" -}} -{{- $ctx.Values.monitoring.enabled | toYaml -}} -{{- else if eq $name "loki-stack" -}} -{{- and $ctx.Values.monitoring.enabled $ctx.Values.monitoring.lokiStack.enabled | toYaml -}} -{{- else if eq $name "node-feature-discovery" -}} -{{- $ctx.Values.nodeFeatureDiscovery.enabled | toYaml -}} -{{- else if eq $name "nvidia-gpu-operator" -}} -{{- $ctx.Values.nvidiaGPUOperator.enabled | toYaml -}} -{{- else if eq $name "mellanox-network-operator" -}} -{{- $ctx.Values.mellanoxNetworkOperator.enabled | toYaml -}} -{{- else if hasKey $ctx.Values.extraAddons $name -}} -{{- dig $name "enabled" true $ctx.Values.extraAddons | toYaml -}} -{{- else -}} -false -{{- end -}} -{{- end }} - -{{/* -Produces the explicit dependencies for an addon. - -The result is returned as an object so it can be used with fromYaml. -*/}} -{{- define "cluster-addons.dependsOn.explicit" -}} -{{- $ctx := index . 0 }} -{{- $name := index . 1 }} -value: - {{- if (list "ccm-openstack" "csi-cinder" | has $name) }} - - cloud-config - {{- else if eq $name "kube-prometheus-stack" }} - - storage - - ingress - {{- else if eq $name "loki-stack" }} - - storage - {{- else if eq $name "nvidia-gpu-operator" }} - - node-feature-discovery - {{- else if eq $name "mellanox-network-operator" }} - - node-feature-discovery - {{- else if hasKey $ctx.Values.extraAddons $name }} - {{- dig $name "dependsOn" list $ctx.Values.extraAddons | toYaml | nindent 2 }} - {{- else }} - [] - {{- end }} -{{- end }} - -{{/* -Produces the dependencies for an addon, resolving any categories and including the -bootstrap addons as an implicit dependency (unless the addon itself is a bootstrap -addon). - -The result is returned as an object so it can be used with fromYaml. -*/}} -{{- define "cluster-addons.dependsOn.all" -}} -{{- $ctx := index . 0 }} -{{- $name := index . 1 }} -{{- $categories := $ctx.Values.categories }} -{{- $explicit := (include "cluster-addons.dependsOn.explicit" . | fromYaml).value | default list }} -value: - {{- if not (has $name $categories.bootstrap) }} - {{- range $categories.bootstrap }} - - {{ . }} - {{- end }} - {{- end }} - {{- range $explicit }} - {{- if hasKey $categories . }} - {{- range (dig . list $categories) }} - - {{ . }} - {{- end }} - {{- else }} - - {{ . }} - {{- end }} - {{- end }} -{{- end }} - -{{/* -Produces the dependencies for an addon, ensuring uniqueness and a consistent ordering. - -The result is returned as an object so it can be used with fromYaml. -*/}} -{{- define "cluster-addons.dependsOn.unique" -}} -{{- $ctx := index . 0 }} -value: {{ - (include "cluster-addons.dependsOn.all" . | fromYaml).value | - default list | - uniq | - sortAlpha | - toYaml | - nindent 2 -}} -{{- end }} - -{{/* -Produces the enabled dependencies for an addon. - -The result is returned as an object so it can be used with fromYaml. -*/}} -{{- define "cluster-addons.dependsOn.enabled" -}} -{{- $ctx := index . 0 }} -{{- $deps := (include "cluster-addons.dependsOn.unique" . | fromYaml).value | default list -}} -value: - {{- range $deps }} - {{- if eq (include "cluster-addons.enabled" (list $ctx .)) "true" }} - - {{ . }} - {{- end }} - {{- end }} -{{- end }} - -{{/* -Produces the uninstall hook weight for the specified addon, ensuring that it is -removed before any of its dependencies. All addons are considered, even those that -are not enabled, because not doing so causes addons to be removed in the wrong -order when two addons with dependencies on each other are uninstalled together. - -Addons with no enabled dependencies have a weight of zero. Addons with at least one -enabled dependency have a weight that is one less than the minimum of the weights -of the dependencies. -*/}} -{{- define "cluster-addons.uninstallHookWeight" -}} -{{- $ctx := index . 0 -}} -{{- $name := index . 1 -}} -{{- $weight := 1 }} -{{- $deps := (include "cluster-addons.dependsOn.unique" . | fromYaml).value | default list -}} -{{- range $deps -}} -{{- $dependencyWeight := include "cluster-addons.uninstallHookWeight" (list $ctx .) | atoi -}} -{{- $weight = min $weight $dependencyWeight -}} -{{- end -}} -{{- sub $weight 1 -}} -{{- end }} diff --git a/charts/cluster-addons/templates/ccm-openstack.yaml b/charts/cluster-addons/templates/ccm-openstack.yaml deleted file mode 100644 index 2627219..0000000 --- a/charts/cluster-addons/templates/ccm-openstack.yaml +++ /dev/null @@ -1,43 +0,0 @@ -{{- define "cluster-addons.ccm-openstack.config" -}} -{{- include "cluster-addons.job.defaults" (list . "ccm-openstack") }} -installType: kustomize -kustomize: - kustomizationTemplate: | - resources: - {{- range .Values.openstack.ccm.manifests }} - - {{ tpl . $ }} - {{- end }} - patches: - - patch: |- - - op: add - path: /spec/template/spec/containers/0/args/- - value: --cluster-name={{ tpl .Values.clusterName . }} - target: - kind: DaemonSet - name: openstack-cloud-controller-manager - {{- if semverCompare "~1.21.0" (tpl .Values.kubernetesVersion . | trimPrefix "v") }} - - patch: |- - - op: add - path: /rules/- - value: - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] - target: - group: rbac.authorization.k8s.io - version: v1 - kind: ClusterRole - name: system:cloud-controller-manager - {{- end }} - {{- with .Values.openstack.ccm.kustomization }} - kustomization: {{ toYaml . | nindent 4 }} - {{- end }} -{{- end }} - -{{- - include "addon.job" (list - . - "ccm-openstack" - "cluster-addons.ccm-openstack.config" - ) -}} diff --git a/charts/cluster-addons/templates/cert-manager.yaml b/charts/cluster-addons/templates/cert-manager.yaml deleted file mode 100644 index 75622b6..0000000 --- a/charts/cluster-addons/templates/cert-manager.yaml +++ /dev/null @@ -1,37 +0,0 @@ -{{- define "cluster-addons.cert-manager.config" -}} -{{- include "cluster-addons.job.defaults" (list . "cert-manager") }} -installType: helm -helm: {{ omit .Values.certManager "enabled" "acmeHttp01Issuer" | toYaml | nindent 2 }} -{{- if and .Values.ingress.enabled .Values.certManager.acmeHttp01Issuer.enabled }} -extraFiles: - acme-http01-issuer.yaml: | - apiVersion: cert-manager.io/v1 - kind: ClusterIssuer - metadata: - name: {{ .Values.certManager.acmeHttp01Issuer.name }} - spec: - acme: - server: {{ .Values.certManager.acmeHttp01Issuer.server }} - privateKeySecretRef: - name: {{ .Values.certManager.acmeHttp01Issuer.name }}-key - solvers: - - http01: - ingress: - {{- if .Values.ingress.nginx.enabled }} - class: {{ dig "controller" "ingressClassResource" "name" "nginx" .Values.ingress.nginx.release.values }} - {{- else }} - {{- fail "Ingress is enabled but no ingress controllers are enabled" }} - {{- end }} -hooks: - postInstall: | - kubectl apply -f ./acme-http01-issuer.yaml -{{- end }} -{{- end }} - -{{- - include "addon.job" (list - . - "cert-manager" - "cluster-addons.cert-manager.config" - ) -}} diff --git a/charts/cluster-addons/templates/cloud-config.yaml b/charts/cluster-addons/templates/cloud-config.yaml deleted file mode 100644 index d60deee..0000000 --- a/charts/cluster-addons/templates/cloud-config.yaml +++ /dev/null @@ -1,101 +0,0 @@ -{{- define "cluster-addons.cloud-config.config" -}} -{{- - $secretNameTemplate := required - ".Values.openstack.cloudCredentialsSecretName is required" - .Values.openstack.cloudCredentialsSecretName -}} -{{- $secretName := tpl $secretNameTemplate . }} -{{- include "cluster-addons.job.defaults" (list . "cloud-config") }} -installType: custom -custom: - install: | - {{- if .Values.clusterApi }} - export EXTERNAL_NETWORK_ID="$( - KUBECONFIG= kubectl get openstackcluster \ - {{ tpl .Values.clusterName . }} \ - --namespace {{ .Release.Namespace }} \ - -o jsonpath='{.status.externalNetwork.id}' - )" - export INTERNAL_NETWORK_NAME="$( - KUBECONFIG= kubectl get openstackcluster \ - {{ tpl .Values.clusterName . }} \ - --namespace {{ .Release.Namespace }} \ - -o jsonpath='{.status.network.name}' - )" - {{- end }} - gomplate --file secret.yaml.tpl | kubectl apply -f - - delete: | - kubectl delete secret --namespace kube-system cloud-config -extraVolumes: - - secret: - name: {{ $secretName }} -extraFiles: - secret.yaml.tpl: | - apiVersion: v1 - kind: Secret - metadata: - name: cloud-config - namespace: kube-system - stringData: - {{ "{{-" }} $clouds := file.Read "./clouds.yaml" {{ "}}" }} - {{ "{{-" }} $cloud := index (data.YAML $clouds).clouds "{{ .Values.openstack.cloudName }}" {{ "}}" }} - clouds.yaml: | - {{ "{{" }} $clouds | indent 4 | trimSpace {{ "}}" }} - {{ "{{-" }} if file.Exists "./cacert" {{ "}}" }} - cacert: | - {{ "{{" }} file.Read "./cacert" | indent 4 | trimSpace {{ "}}" }} - {{ "{{-" }} end {{ "}}" }} - cloud.conf: | - [Global] - use-clouds=true - clouds-file=/etc/config/clouds.yaml - cloud={{ .Values.openstack.cloudName }} - {{ "{{-" }} if file.Exists "./cacert" {{ "}}" }} - ca-file=/etc/config/cacert - {{ "{{-" }} end {{ "}}" }} - {{ "{{-" }} if has $cloud "verify" {{ "}}" }} - {{ "{{-" }} if not $cloud.verify {{ "}}" }} - tls-insecure=true - {{ "{{-" }} end {{ "}}" }} - {{ "{{-" }} end {{ "}}" }} - [Networking] - {{- $networkingItems := default dict .Values.openstack.cloudConfig.Networking }} - {{- if hasKey $networkingItems "internal-network-name" }} - internal-network-name={{ index $networkingItems "internal-network-name" }} - {{- else if .Values.clusterApi }} - internal-network-name={{ "{{" }} .Env.INTERNAL_NETWORK_NAME {{ "}}" }} - {{- end }} - {{- range $netName, $netValue := omit $networkingItems "internal-network-name" }} - {{ $netName }}={{ $netValue }} - {{- end }} - [LoadBalancer] - {{- $lbItems := default dict .Values.openstack.cloudConfig.LoadBalancer }} - {{- if hasKey $lbItems "floating-network-id" }} - floating-network-id={{ index $lbItems "floating-network-id" }} - {{- else if .Values.clusterApi }} - floating-network-id={{ "{{" }} .Env.EXTERNAL_NETWORK_ID {{ "}}" }} - {{- end }} - {{- range $lbName, $lbValue := omit $lbItems "floating-network-id" }} - {{ $lbName }}={{ $lbValue }} - {{- end }} - {{- - range $section, $items := omit - .Values.openstack.cloudConfig - "Global" - "LoadBalancer" - "Networking" - }} - [{{ $section }}] - {{- range $name, $value := $items }} - {{ $name }}={{ $value }} - {{- end }} - {{ end }} -{{- end }} - -{{- - include "addon.job" (list - . - "cloud-config" - "cluster-addons.cloud-config.config" - ) -}} diff --git a/charts/cluster-addons/templates/cni-calico.yaml b/charts/cluster-addons/templates/cni-calico.yaml deleted file mode 100644 index 095772b..0000000 --- a/charts/cluster-addons/templates/cni-calico.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- define "cluster-addons.cni-calico.config" -}} -{{- include "cluster-addons.job.defaults" (list . "cni-calico") }} -installType: helm -helm: {{ omit .Values.cni.calico "installation" | toYaml | nindent 2 }} -extraFiles: - installation.yaml: | - apiVersion: operator.tigera.io/v1 - kind: Installation - metadata: - name: default - spec: {{ toYaml .Values.cni.calico.installation | nindent 6 }} -hooks: - postInstall: | - KUBEADM_POD_CIDR="$( - kubectl -n kube-system get configmap kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' | \ - yq '.networking.podSubnet' - )" - sed "s#__KUBEADM_POD_CIDR__#${KUBEADM_POD_CIDR}#" installation.yaml | kubectl apply -f - - preDelete: | - kubectl delete installations.operator.tigera.io default || true -{{- end }} - -{{- - include "addon.job" (list - . - "cni-calico" - "cluster-addons.cni-calico.config" - ) -}} diff --git a/charts/cluster-addons/templates/cni-cilium.yaml b/charts/cluster-addons/templates/cni-cilium.yaml deleted file mode 100644 index cf5c16c..0000000 --- a/charts/cluster-addons/templates/cni-cilium.yaml +++ /dev/null @@ -1,44 +0,0 @@ -{{- define "cluster-addons.cni-cilium.config" -}} -{{- include "cluster-addons.job.defaults" (list . "cni-cilium") }} -installType: helm -helm: {{ toYaml .Values.cni.cilium | nindent 2 }} -{{- $kubeProxyReplacement := dig "kubeProxyReplacement" "probe" .Values.cni.cilium.release.values }} -{{- if eq $kubeProxyReplacement "strict" }} -{{- $hasServiceHost := hasKey .Values.cni.cilium.release.values "k8sServiceHost" }} -{{- $hasServicePort := hasKey .Values.cni.cilium.release.values "k8sServicePort" }} -{{- if or (and $hasServiceHost $hasServicePort) .Values.kubeconfigSecret.name }} -{{- if not (and $hasServiceHost $hasServicePort) }} -hooks: - {{/* Point Cilium at the Kubernetes server targetted by the kubeconfig file */}} - preInstall: | - SERVER="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')" - SCHEME="$(echo "$SERVER" | cut -d':' -f1)" - ADDRESS="$(echo "$SERVER" | cut -d'/' -f3)" - HOST="$(echo "$ADDRESS" | cut -d':' -f1)" - if grep -q ":" <<< "$ADDRESS"; then - PORT="$(echo "$ADDRESS" | cut -d':' -f2)" - else - if [ "$SCHEME" = "http" ]; then - PORT=80 - else - PORT=443 - fi - fi - - HELM_EXTRA_ARGS="--set k8sServiceHost=$HOST" - HELM_EXTRA_ARGS="$HELM_EXTRA_ARGS --set k8sServicePort=$PORT" - export HELM_EXTRA_ARGS -{{- end }} -{{- else }} -{{- fail "k8sServiceHost and k8sServicePort must be specified when using a service account with kubeProxyReplacement=strict" }} -{{- end }} -{{- end }} -{{- end }} - -{{- - include "addon.job" (list - . - "cni-cilium" - "cluster-addons.cni-cilium.config" - ) -}} diff --git a/charts/cluster-addons/templates/csi-cinder.yaml b/charts/cluster-addons/templates/csi-cinder.yaml deleted file mode 100644 index 9bf1844..0000000 --- a/charts/cluster-addons/templates/csi-cinder.yaml +++ /dev/null @@ -1,50 +0,0 @@ -{{- define "cluster-addons.csi-cinder.config" -}} -{{- include "cluster-addons.job.defaults" (list . "csi-cinder") }} -installType: kustomize -kustomize: - kustomizationTemplate: | - resources: - {{- range .Values.openstack.csiCinder.manifests }} - - {{ tpl . $ }} - {{- end }} - {{- if .Values.openstack.csiCinder.storageClass.enabled }} - - ./storageclass.yaml - {{- end }} - {{- with .Values.openstack.csiCinder.kustomization }} - kustomization: {{ toYaml . | nindent 4 }} - {{- end }} -{{- with .Values.openstack.csiCinder.storageClass }} -{{- if .enabled }} -extraFiles: - storageclass.yaml: | - apiVersion: storage.k8s.io/v1 - kind: StorageClass - metadata: - name: {{ .name }} - {{- if .isDefault }} - annotations: - storageclass.kubernetes.io/is-default-class: "true" - {{- end }} - provisioner: cinder.csi.openstack.org - parameters: - availability: {{ .availabilityZone }} - {{- with .volumeType }} - type: {{ . }} - {{- end }} - reclaimPolicy: {{ .reclaimPolicy }} - allowVolumeExpansion: {{ .allowVolumeExpansion }} - volumeBindingMode: WaitForFirstConsumer - {{- with .allowedTopologies }} - allowedTopologies: {{ toYaml . | nindent 6 }} - {{- end }} -{{- end }} -{{- end }} -{{- end }} - -{{- - include "addon.job" (list - . - "csi-cinder" - "cluster-addons.csi-cinder.config" - ) -}} diff --git a/charts/cluster-addons/templates/extra-addons.yaml b/charts/cluster-addons/templates/extra-addons.yaml deleted file mode 100644 index d856125..0000000 --- a/charts/cluster-addons/templates/extra-addons.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- define "cluster-addons.extra-addons.config" -}} -{{- $ctx := index . 0 }} -{{- $name := index . 1 }} -{{- - $config := omit - (index . 2) - "kubeconfigSecret" - "serviceAccountName" - "enabled" - "dependsOn" - "uninstallHookWeight" - "extraInitContainers" -}} -{{- - include "cluster-addons.job.defaults" (list $ctx $name) | - fromYaml | - merge $config | - toYaml -}} -{{- end }} - -{{- range $name, $config := .Values.extraAddons }} ---- -{{- $merged := include "cluster-addons.extra-addons.config" (list $ $name $config) | fromYaml }} -{{- include "addon.job.fromConfig" (list $ $name $merged) }} -{{- end }} diff --git a/charts/cluster-addons/templates/ingress-nginx.yaml b/charts/cluster-addons/templates/ingress-nginx.yaml deleted file mode 100644 index b63fe4a..0000000 --- a/charts/cluster-addons/templates/ingress-nginx.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- define "cluster-addons.ingress-nginx.config" -}} -{{- include "cluster-addons.job.defaults" (list . "ingress-nginx") }} -installType: helm -helm: {{ omit .Values.ingress.nginx "enabled" | toYaml | nindent 2 }} -{{- end }} - -{{- - include "addon.job" (list - . - "ingress-nginx" - "cluster-addons.ingress-nginx.config" - ) -}} diff --git a/charts/cluster-addons/templates/kube-prometheus-stack.yaml b/charts/cluster-addons/templates/kube-prometheus-stack.yaml deleted file mode 100644 index 4c2e9b0..0000000 --- a/charts/cluster-addons/templates/kube-prometheus-stack.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{- define "cluster-addons.kube-prometheus-stack.config" -}} -{{- include "cluster-addons.job.defaults" (list . "kube-prometheus-stack") }} -installType: helm -helm: {{ toYaml .Values.monitoring.kubePrometheusStack | nindent 2 }} -extraFiles: - configmap-nvidia-dcgm-exporter-dashboard.yaml: | - apiVersion: v1 - kind: ConfigMap - metadata: - name: nvidia-dcgm-exporter-dashboard - namespace: {{ .Values.monitoring.kubePrometheusStack.release.namespace }} - labels: - {{- include "cluster-addons.labels" . | nindent 8 }} - grafana_dashboard: "1" - data: - nvidia-dcgm-exporter-dashboard.json: | - {{- .Files.Get "grafana-dashboards/nvidia-dcgm-exporter-dashboard_rev2.json" | nindent 8 }} -hooks: - postInstall: | - kubectl apply -f ./configmap-nvidia-dcgm-exporter-dashboard.yaml - preDelete: | - kubectl delete -f ./configmap-nvidia-dcgm-exporter-dashboard.yaml -{{- end }} - -{{- - include "addon.job" (list - . - "kube-prometheus-stack" - "cluster-addons.kube-prometheus-stack.config" - ) -}} diff --git a/charts/cluster-addons/templates/kubernetes-dashboard.yaml b/charts/cluster-addons/templates/kubernetes-dashboard.yaml deleted file mode 100644 index 7ac05c8..0000000 --- a/charts/cluster-addons/templates/kubernetes-dashboard.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- define "cluster-addons.kubernetes-dashboard.config" -}} -{{- include "cluster-addons.job.defaults" (list . "kubernetes-dashboard") }} -installType: helm -helm: {{ omit .Values.kubernetesDashboard "enabled" | toYaml | nindent 2 }} -{{- end }} - -{{- - include "addon.job" (list - . - "kubernetes-dashboard" - "cluster-addons.kubernetes-dashboard.config" - ) -}} diff --git a/charts/cluster-addons/templates/loki-stack.yaml b/charts/cluster-addons/templates/loki-stack.yaml deleted file mode 100644 index 50c0e52..0000000 --- a/charts/cluster-addons/templates/loki-stack.yaml +++ /dev/null @@ -1,52 +0,0 @@ -{{- define "cluster-addons.loki-stack.config" -}} -{{- include "cluster-addons.job.defaults" (list . "loki-stack") }} -installType: helm -helm: {{ omit .Values.monitoring.lokiStack "enabled" | toYaml | nindent 2 }} -# Add a datasource to the kube-prometheus-stack Grafana -extraFiles: - configmap-grafana-datasource.yaml: | - apiVersion: v1 - kind: ConfigMap - metadata: - name: loki-stack-grafana-datasource - namespace: {{ .Values.monitoring.kubePrometheusStack.release.namespace }} - labels: - {{- include "cluster-addons.labels" . | nindent 8 }} - grafana_datasource: "1" - data: - loki-datasource.yaml: |- - apiVersion: 1 - datasources: - - name: Loki - type: loki - url: http://loki-stack.{{ .Values.monitoring.lokiStack.release.namespace }}:3100 - access: proxy - version: 1 - configmap-grafana-dashboard.yaml: | - apiVersion: v1 - kind: ConfigMap - metadata: - name: loki-stack-grafana-dashboard - namespace: {{ .Values.monitoring.kubePrometheusStack.release.namespace }} - labels: - {{- include "cluster-addons.labels" . | nindent 8 }} - grafana_dashboard: "1" - data: - loki-dashboard.json: | - {{- .Files.Get "grafana-dashboards/loki-dashboard.json" | nindent 8 }} -hooks: - postInstall: | - kubectl apply -f ./configmap-grafana-datasource.yaml - kubectl apply -f ./configmap-grafana-dashboard.yaml - preDelete: | - kubectl delete -f ./configmap-grafana-datasource.yaml - kubectl delete -f ./configmap-grafana-dashboard.yaml -{{- end }} - -{{- - include "addon.job" (list - . - "loki-stack" - "cluster-addons.loki-stack.config" - ) -}} diff --git a/charts/cluster-addons/templates/mellanox-network-operator.yaml b/charts/cluster-addons/templates/mellanox-network-operator.yaml deleted file mode 100644 index df1e16a..0000000 --- a/charts/cluster-addons/templates/mellanox-network-operator.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- define "cluster-addons.mellanox-network-operator.config" -}} -{{- include "cluster-addons.job.defaults" (list . "mellanox-network-operator") }} -installType: helm -helm: {{ omit .Values.mellanoxNetworkOperator "enabled" | toYaml | nindent 2 }} -{{- end }} - -{{- - include "addon.job" (list - . - "mellanox-network-operator" - "cluster-addons.mellanox-network-operator.config" - ) -}} diff --git a/charts/cluster-addons/templates/metrics-server.yaml b/charts/cluster-addons/templates/metrics-server.yaml deleted file mode 100644 index 8b7c71d..0000000 --- a/charts/cluster-addons/templates/metrics-server.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- define "cluster-addons.metrics-server.config" -}} -{{- include "cluster-addons.job.defaults" (list . "metrics-server") }} -installType: kustomize -kustomize: - kustomizationTemplate: | - resources: - {{- range .Values.metricsServer.manifests }} - - {{ tpl . $ }} - {{- end }} - {{- with .Values.metricsServer.kustomization }} - kustomization: {{ toYaml . | nindent 4 }} - {{- end }} -{{- end }} - -{{- - include "addon.job" (list - . - "metrics-server" - "cluster-addons.metrics-server.config" - ) -}} diff --git a/charts/cluster-addons/templates/nfd.yaml b/charts/cluster-addons/templates/nfd.yaml deleted file mode 100644 index f7b963e..0000000 --- a/charts/cluster-addons/templates/nfd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- define "cluster-addons.node-feature-discovery.config" -}} -{{- include "cluster-addons.job.defaults" (list . "node-feature-discovery") }} -installType: helm -helm: {{ omit .Values.nodeFeatureDiscovery "enabled" | toYaml | nindent 2 }} -{{- end }} - -{{- - include "addon.job" (list - . - "node-feature-discovery" - "cluster-addons.node-feature-discovery.config" - ) -}} diff --git a/charts/cluster-addons/templates/nvidia-gpu-operator.yaml b/charts/cluster-addons/templates/nvidia-gpu-operator.yaml deleted file mode 100644 index 5c9e0a8..0000000 --- a/charts/cluster-addons/templates/nvidia-gpu-operator.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- define "cluster-addons.nvidia-gpu-operator.config" -}} -{{- include "cluster-addons.job.defaults" (list . "nvidia-gpu-operator") }} -installType: helm -helm: {{ omit .Values.nvidiaGPUOperator "enabled" | toYaml | nindent 2 }} -{{- end }} - -{{- - include "addon.job" (list - . - "nvidia-gpu-operator" - "cluster-addons.nvidia-gpu-operator.config" - ) -}} diff --git a/charts/cluster-addons/templates/prometheus-operator-crds.yaml b/charts/cluster-addons/templates/prometheus-operator-crds.yaml deleted file mode 100644 index d164009..0000000 --- a/charts/cluster-addons/templates/prometheus-operator-crds.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- define "cluster-addons.prometheus-operator-crds.config" -}} -{{- include "cluster-addons.job.defaults" (list . "prometheus-operator-crds") }} -installType: custom -custom: - install: | - crd_apply() { kubectl replace -f "$1" || kubectl create -f "$1"; } - {{- range .Values.monitoring.prometheusOperatorCrds }} - crd_apply {{ tpl . $ }} - {{- end }} - delete: | - {{- range .Values.monitoring.prometheusOperatorCrds }} - kubectl delete -f {{ tpl . $ }} - {{- end }} -{{- end }} - -{{- - include "addon.job" (list - . - "prometheus-operator-crds" - "cluster-addons.prometheus-operator-crds.config" - ) -}} diff --git a/charts/cluster-addons/templates/purge-cloud-resources.yaml b/charts/cluster-addons/templates/purge-cloud-resources.yaml deleted file mode 100644 index b821d12..0000000 --- a/charts/cluster-addons/templates/purge-cloud-resources.yaml +++ /dev/null @@ -1,102 +0,0 @@ -{{- if and .Values.clusterApi .Values.openstack.enabled }} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ printf "%s-%s" (include "cluster-addons.fullname" .) "purge-cloud-resources" | trunc 63 | trimSuffix "-" }} - labels: {{ include "cluster-addons.labels" . | nindent 4 }} - annotations: - helm.sh/hook: pre-delete - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - backoffLimit: {{ .Values.jobDefaults.backoffLimit }} - activeDeadlineSeconds: {{ .Values.jobDefaults.activeDeadlineSeconds }} - template: - metadata: - labels: {{ include "cluster-addons.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.jobDefaults.imagePullSecrets }} - imagePullSecrets: {{ toYaml . | nindent 8 }} - {{- end }} - securityContext: {{ toYaml .Values.jobDefaults.podSecurityContext | nindent 8 }} - restartPolicy: OnFailure - serviceAccountName: {{ tpl .Values.serviceAccount.name . }} - {{- if .Values.kubeconfigSecret.name }} - # Use an init container to install the kubeconfig file from the specified secret if required - # We don't use a regular volume for this because we need the hook not to block in the case - # where the secret is not available - initContainers: - - name: install-kubeconfig - image: {{ - printf "%s:%s" - .Values.jobDefaults.image.repository - (default .Chart.AppVersion .Values.jobDefaults.image.tag) - }} - imagePullPolicy: {{ .Values.jobDefaults.image.pullPolicy }} - securityContext: {{ toYaml .Values.jobDefaults.securityContext | nindent 12 }} - args: - - /bin/bash - - -c - - | - set -ex - get_kubeconfig() { - kubectl get secret {{ tpl .Values.kubeconfigSecret.name . }} \ - -n {{ .Release.Namespace }} \ - -o go-template='{{ printf "{{ index .data \"%s\" | base64decode }}" .Values.kubeconfigSecret.key }}' \ - > /config/auth/kubeconfig - } - get_kubeconfig || true - resources: {{ toYaml .Values.jobDefaults.resources | nindent 12 }} - volumeMounts: - - name: kubeconfig - mountPath: /config/auth - {{- end }} - containers: - - name: purge-cloud-resources - image: {{ - printf "%s:%s" - .Values.jobDefaults.image.repository - (default .Chart.AppVersion .Values.jobDefaults.image.tag) - }} - imagePullPolicy: {{ .Values.jobDefaults.image.pullPolicy }} - securityContext: {{ toYaml .Values.jobDefaults.securityContext | nindent 12 }} - # We can only make a best effort to delete the resources as we don't want the hook to block - # So we bail without an error if the kubeconfig doesn't exist, the API is not reachable or - # the deletion fails - args: - - /bin/bash - - -c - - | - set -x - {{- if .Values.kubeconfigSecret.name }} - test -f "$KUBECONFIG" || exit 0 - {{- end }} - kubectl version || exit 0 - for ns in $(kubectl get ns -o jsonpath='{.items[*].metadata.name}'); do - for svc in $(kubectl get svc -n "$ns" -o jsonpath='{.items[?(@.spec.type == "LoadBalancer")].metadata.name}'); do - kubectl delete svc "$svc" -n "$ns" || true - done - done - {{- if .Values.kubeconfigSecret.name }} - env: - - name: KUBECONFIG - value: /config/auth/kubeconfig - {{- end }} - resources: {{ toYaml .Values.jobDefaults.resources | nindent 12 }} - volumeMounts: - - name: kubeconfig - mountPath: /config/auth - readOnly: true - hostNetwork: {{ .Values.jobDefaults.hostNetwork }} - {{- with .Values.jobDefaults.nodeSelector }} - nodeSelector: {{ toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.jobDefaults.affinity }} - affinity: {{ toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.jobDefaults.tolerations }} - tolerations: {{ toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: kubeconfig - emptyDir: {} -{{- end }} diff --git a/charts/cluster-addons/templates/rbac/clusterrolebinding.yaml b/charts/cluster-addons/templates/rbac/clusterrolebinding.yaml deleted file mode 100644 index 601debc..0000000 --- a/charts/cluster-addons/templates/rbac/clusterrolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if and .Values.serviceAccount.create (not .Values.kubeconfigSecret.name) -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "cluster-addons.fullname" . }} - labels: {{ include "cluster-addons.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - namespace: {{ .Release.Namespace }} - name: {{ tpl .Values.serviceAccount.name . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ .Values.serviceAccount.clusterRoleName }} -{{- end }} diff --git a/charts/cluster-addons/templates/rbac/role.yaml b/charts/cluster-addons/templates/rbac/role.yaml deleted file mode 100644 index 888ce14..0000000 --- a/charts/cluster-addons/templates/rbac/role.yaml +++ /dev/null @@ -1,44 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "cluster-addons.fullname" . }}-manage-jobs - labels: {{ include "cluster-addons.labels" . | nindent 4 }} -rules: - - apiGroups: - - "" - resources: - - secrets - verbs: - - list - - get - - apiGroups: - - batch - resources: - - jobs - verbs: - - list - - get - - watch - - patch - {{- if .Values.clusterApi }} - - apiGroups: - - cluster.x-k8s.io - resources: - - clusters - verbs: - - list - - get - - watch - {{- if .Values.openstack.enabled }} - - apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - openstackclusters - verbs: - - list - - get - - watch - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/cluster-addons/templates/rbac/rolebinding.yaml b/charts/cluster-addons/templates/rbac/rolebinding.yaml deleted file mode 100644 index 7be189a..0000000 --- a/charts/cluster-addons/templates/rbac/rolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "cluster-addons.fullname" . }}-manage-jobs - labels: {{ include "cluster-addons.labels" . | nindent 4 }} -subjects: - - kind: ServiceAccount - namespace: {{ .Release.Namespace }} - name: {{ tpl .Values.serviceAccount.name . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "cluster-addons.fullname" . }}-manage-jobs -{{- end }} diff --git a/charts/cluster-addons/templates/rbac/serviceaccount.yaml b/charts/cluster-addons/templates/rbac/serviceaccount.yaml deleted file mode 100644 index 919fd92..0000000 --- a/charts/cluster-addons/templates/rbac/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ tpl .Values.serviceAccount.name . }} - labels: {{ include "cluster-addons.labels" . | nindent 4 }} -{{- end }} diff --git a/charts/cluster-addons/values.yaml b/charts/cluster-addons/values.yaml deleted file mode 100644 index 9de55e8..0000000 --- a/charts/cluster-addons/values.yaml +++ /dev/null @@ -1,429 +0,0 @@ -# The name of the Kubernetes cluster we are deploying to -# Defaults to the release name if not given for use as a dependency of openstack-cluster -clusterName: "{{ .Release.Name }}" - -# The Kubernetes version of the target cluster -# This is treated as a template at rendering time -kubernetesVersion: v1.22 - -# Indicates whether the addons are being deployed as part of a Cluster API cluster -# If true then addons will wait for the cluster to become ready before installing, except -# for the bootstrap addons which just wait for the API to become available -clusterApi: false - -# Details of a secret containing a kubeconfig file for a remote cluster -# If given, this is used in preference to a service account -kubeconfigSecret: - # The name of the secret - # This is treated as a template during rendering - name: - # The key of the kubeconfig file in the secret - key: value - -# Options for the service account to use -# A pre-existing service account can be used, or a new one can be created -# -# A service account is always required as it is used by the pre-delete hook -# to suspend any install jobs that are still running prior to running the deletion -# -# The permissions required by the service account depend on whether the installation -# is targetting a remote cluster or the local cluster -# -# Whether the installation target is local or remote, the service account needs to -# have permission to list and patch jobs in the release namespace for the delete hook -# in order to suspend any running install jobs -# -# When the installation targets the local cluster, the service account must also have -# permission to create any resources that need to be installed, which could be into -# other namespaces - the cluster-admin cluster role is normally used for this -serviceAccount: - # Indicates whether to create a new service account - create: true - # The name of the cluster role to bind the created service account to - clusterRoleName: cluster-admin - # The name of the service account - # If create = true, this is the name of the created service account - # If create = false, this is the name of an existing service account to use - # This is treated as a template during rendering - name: "{{ include \"cluster-addons.fullname\" . }}-deployer" - -# Default settings for jobs -jobDefaults: - image: - repository: ghcr.io/stackhpc/k8s-utils - tag: # Defaults to chart appVersion if not given - pullPolicy: IfNotPresent - imagePullSecrets: [] - backoffLimit: 1000 - activeDeadlineSeconds: 3600 - podSecurityContext: - runAsNonRoot: true - securityContext: - allowPrivilegeEscalation: false - resources: {} - hostNetwork: false - tolerations: [] - nodeSelector: {} - affinity: {} - -# The available categories for dependencies and the addons that belong to them -categories: - bootstrap: - - cloud-config - - ccm-openstack - - cni-calico - - cni-cilium - - prometheus-operator-crds - storage: [csi-cinder] - ingress: [ingress-nginx] - -# Settings for the CNI addon -cni: - # Indicates if a CNI should be deployed - enabled: true - # The type of CNI to deploy - supported values are calico or cilium - type: calico - # Settings for the calico CNI - calico: - chart: - repo: https://projectcalico.docs.tigera.io/charts - name: tigera-operator - version: v3.23.3 - release: - namespace: tigera-operator - # See https://projectcalico.docs.tigera.io/getting-started/kubernetes/helm - values: - # Managing the installation separately makes deriving the pod CIDR cleaner - installation: - enabled: false - # The spec of the Calico installation - # See https://projectcalico.docs.tigera.io/reference/installation/api - installation: - calicoNetwork: - # By default, disable BGP - bgp: Disabled - # Use the interface that holds the Kubernetes internal IP - nodeAddressAutodetectionV4: - kubernetes: NodeInternalIP - # Use a single IP pool with VXLAN - # The special variable __KUBEADM_POD_CIDR__ is replaced with the pod CIDR from the - # kubeadm configmap, if kubeadm is in use - ipPools: - - cidr: __KUBEADM_POD_CIDR__ - encapsulation: VXLAN - # Settings for the Cilium CNI - cilium: - chart: - repo: https://helm.cilium.io/ - name: cilium - version: 1.11.1 - release: - namespace: kube-system - # See https://docs.cilium.io/en/stable/gettingstarted/k8s-install-helm/ for details - values: - ipam: - mode: kubernetes - -# Settings for the OpenStack integrations -openstack: - # Indicates if the OpenStack integrations should be enabled - enabled: false - # The version of the OpenStack cloud provider to install - # By default, use the release branch for the Kubernetes version of the target cluster - version: release-{{ tpl .Values.kubernetesVersion . | trimPrefix "v" }} - # The base URL for OpenStack cloud provider manifests - # By default, pull the manifests from GitHub at the specified version - manifestsBaseURL: https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/{{ tpl .Values.openstack.version . }} - # The name of a secret containing a clouds.yaml file and optional cacert - # If the cacert is present, it should be referred to in the clouds.yaml file as /etc/config/cacert - # See https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html#ssl-settings - cloudCredentialsSecretName: - # The name of the cloud to use in the clouds.yaml - cloudName: openstack - # cloud-config options for the OpenStack integrations - # The [Global] section is configured to use the specified cloud from .Values.clouds - # See https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md#config-openstack-cloud-controller-manager - # and https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md#block-storage - cloudConfig: - # By default, ignore volume AZs for Cinder as most clouds have a single globally-attachable Cinder AZ - BlockStorage: - ignore-volume-az: true - # Settings for the Cloud Controller Manager (CCM) - ccm: - # Indicates if the OpenStack CCM should be enabled - # By default, the CCM is enabled if the OpenStack integrations are enabled - enabled: true - # The prefix for RBAC manifests - # Unfortunately, this changes for different Kubernetes versions - rbacManifestsPrefix: >- - {{ - tpl .Values.kubernetesVersion . | - trimPrefix "v" | - semverCompare ">=1.22" | - ternary "manifests/controller-manager" "cluster/addons/rbac" - }} - # The URLs to use for the manifests - manifests: - - "{{ tpl .Values.openstack.manifestsBaseURL . }}/{{ tpl .Values.openstack.ccm.rbacManifestsPrefix . }}/cloud-controller-manager-roles.yaml" - - "{{ tpl .Values.openstack.manifestsBaseURL . }}/{{ tpl .Values.openstack.ccm.rbacManifestsPrefix . }}/cloud-controller-manager-role-bindings.yaml" - - "{{ tpl .Values.openstack.manifestsBaseURL . }}/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml" - # Any kustomization to apply to the OpenStack CCM manifests - kustomization: {} - # Settings for the Cinder CSI plugin - csiCinder: - # Indicates if the Cinder CSI should be enabled - # By default, it is enabled if the OpenStack integrations are enabled - enabled: true - # The URLs to use for the manifests - manifests: - - "{{ tpl .Values.openstack.manifestsBaseURL . }}/manifests/cinder-csi-plugin/cinder-csi-controllerplugin-rbac.yaml" - - "{{ tpl .Values.openstack.manifestsBaseURL . }}/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml" - - "{{ tpl .Values.openstack.manifestsBaseURL . }}/manifests/cinder-csi-plugin/cinder-csi-nodeplugin-rbac.yaml" - - "{{ tpl .Values.openstack.manifestsBaseURL . }}/manifests/cinder-csi-plugin/cinder-csi-nodeplugin.yaml" - - "{{ tpl .Values.openstack.manifestsBaseURL . }}/manifests/cinder-csi-plugin/csi-cinder-driver.yaml" - # Any kustomization to apply to the OpenStack CCM manifests - kustomization: {} - # Variables affecting the definition of the storage class - storageClass: - # Indicates if the storage class should be enabled - enabled: true - # The name of the storage class - name: csi-cinder - # Indicates if the storage class should be annotated as the default storage class - isDefault: true - # The reclaim policy for the storage class - reclaimPolicy: Delete - # Indicates if volume expansion is allowed - allowVolumeExpansion: true - # The Cinder availability zone to use for volumes provisioned by the storage class - availabilityZone: nova - # The Cinder volume type to use for volumes provisioned by the storage class - # If not given, the default volume type will be used - volumeType: - # The allowed topologies for the storage class - allowedTopologies: - -# Settings for the metrics server -metricsServer: - # Indicates if the metrics server should be deployed - enabled: true - # The version of the metrics server to deploy - version: v0.6.1 - # The URLs of the metrics server manifests - manifests: - - https://github.com/kubernetes-sigs/metrics-server/releases/download/{{ .Values.metricsServer.version }}/components.yaml - # Any kustomization to be applied to the metrics server manifests - kustomization: - patches: - - patch: |- - - op: add - path: /spec/template/spec/containers/0/args/- - value: --kubelet-insecure-tls - target: - kind: Deployment - name: metrics-server - -# Settings for the Kubernetes dashboard -kubernetesDashboard: - # Indicates if the Kubernetes dashboard should be enabled - enabled: false - chart: - repo: https://kubernetes.github.io/dashboard - name: kubernetes-dashboard - version: 5.3.1 - release: - namespace: kubernetes-dashboard - values: - # Enable the metrics scraper by default - metricsScraper: - enabled: true - -# Settings for cert-manager -certManager: - # Indicates if cert-manager should be enabled - enabled: false - chart: - repo: https://charts.jetstack.io - name: cert-manager - version: v1.5.5 - release: - namespace: cert-manager - # See https://cert-manager.io/docs/installation/helm/ for available values - values: - # By default, make sure the cert-manager CRDs are installed - installCRDs: true - # Disable Prometheus support for now - prometheus: - enabled: false - # Settings for automatic ACME HTTP01 support using Let's Encrypt - # This is only enabled if ingress is also enabled - acmeHttp01Issuer: - enabled: yes - name: letsencrypt-http01 - server: https://acme-v02.api.letsencrypt.org/directory - -# Settings for ingress controllers -ingress: - # Indicates if ingress controllers should be enabled - enabled: false - # Settings for the Nginx ingress controller - nginx: - # Indicates if the Nginx ingress controller should be enabled - enabled: true - chart: - repo: https://kubernetes.github.io/ingress-nginx - name: ingress-nginx - version: 4.0.18 - release: - namespace: ingress-nginx - # See https://github.com/kubernetes/ingress-nginx/tree/main/charts/ingress-nginx#configuration - values: {} - -# Settings for cluster monitoring -monitoring: - # Indicates if the cluster monitoring should be enabled - enabled: false - prometheusOperatorCrds: - - https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml - - https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml - - https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml - - https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml - - https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml - - https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml - - https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml - - https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml - kubePrometheusStack: - chart: - repo: https://prometheus-community.github.io/helm-charts - name: kube-prometheus-stack - version: 34.6.0 - release: - namespace: monitoring-system - values: {} - lokiStack: - enabled: true - chart: - repo: https://grafana.github.io/helm-charts - name: loki-stack - version: 2.6.1 - release: - namespace: monitoring-system - values: {} - -# Settings for node feature discovery -nodeFeatureDiscovery: - # Indicates if node feature discovery should be enabled - enabled: true - chart: - repo: https://kubernetes-sigs.github.io/node-feature-discovery/charts - name: node-feature-discovery - version: 0.11.0 - release: - namespace: node-feature-discovery - values: - master: - extraLabelNs: - - nvidia.com - worker: - # Allow the NFD pods to be scheduled on master nodes - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "" - effect: "NoSchedule" - - key: "nvidia.com/gpu" - operator: "Equal" - value: "present" - effect: "NoSchedule" - # We want to be able to identify nodes with high-performance hardware - # So the whitelisted device classes are: - # 02 - Network Controllers (e.g. Ethernet, Infiniband) - # 03 - Display Controllers (e.g. GPUs) - # 0b40 - Co-processors - # 12 - Processing Accelerators (e.g. specialised AI inference chips) - config: - sources: - pci: - deviceClassWhitelist: - - "02" - - "03" - - "0b40" - - "12" - deviceLabelFields: - - vendor - -# Settings for the NVIDIA GPU operator -nvidiaGPUOperator: - # Indicates if the NVIDIA GPU operator should be enabled - # Note that because it uses node feature discovery to run only on nodes - # with an NVIDIA GPU available, the overhead of enabling this on clusters - # that do not need it now but may need it in the future is low - enabled: true - chart: - repo: https://nvidia.github.io/gpu-operator - name: gpu-operator - version: v1.10.0 - release: - namespace: gpu-operator - values: - # Use the shared NFD - nfd: - enabled: false - # Export operator and node metrics in a Prometheus format. - # The component provides information on the status of the - # operator (e.g. reconciliation status, number of GPU enabled nodes). - nodeStatusExporter: - enabled: true - toolkit: - # Allowing the toolkit to edit /etc/containerd/config.toml (the default) - # breaks nvidia pod deployment on clusters with Harbor cache enabled. - # Instead make a new config file specifically for nvidia runtime config, - # which is parsed as an "include" in the main containerd config file. - # - # https://github.com/NVIDIA/gpu-operator/issues/301 - env: - - name: "CONTAINERD_CONFIG" - value: "/etc/containerd/conf.d/nvidia.toml" - -# Settings for the Mellanox network operator -mellanoxNetworkOperator: - # Indicates if the network operator should be enabled - # Note that because it uses node feature discovery to run only on nodes - # with a Mellanox NIC available, the overhead of enabling this on clusters - # that do not need it now but may need it in the future is low - enabled: true - chart: - repo: https://mellanox.github.io/network-operator - name: network-operator - version: 1.1.0 - release: - namespace: network-operator - values: - # Use the shared NFD - nfd: - enabled: false - # Deploy the default NICClusterPolicy - deployCR: true - # Deploy the OFED driver onto nodes with a suitable NIC - ofedDriver: - deploy: true - # OFED takes ages to deploy on low-resource nodes - # The startup probe has a fixed failure threshold of 60 - # So in order to give the drivers up to one hour to install, we use a period - # of 60 seconds for the startup probe - startupProbe: - initialDelaySeconds: 60 - periodSeconds: 60 - # Deploy the RDMA shared device plugin to allow pods to access the RDMA device - rdmaSharedDevicePlugin: - deploy: true - # Disable all other features for now - sriovNetworkOperator: - enabled: false - sriovDevicePlugin: - deploy: false - secondaryNetwork: - deploy: false - -# Map of extra addons in the form "component name" -> "addon spec" -extraAddons: {} diff --git a/charts/openstack-cluster/Chart.yaml b/charts/openstack-cluster/Chart.yaml index 56f24df..283c47a 100644 --- a/charts/openstack-cluster/Chart.yaml +++ b/charts/openstack-cluster/Chart.yaml @@ -4,10 +4,3 @@ description: Helm chart for deploying a cluster on an OpenStack cloud using Clus type: application version: 0.1.0 appVersion: main - -dependencies: - - name: cluster-addons - version: ">=0-0" - repository: file://../cluster-addons - alias: addons - condition: addons.enabled diff --git a/charts/cluster-addons/grafana-dashboards/loki-dashboard.json b/charts/openstack-cluster/grafana-dashboards/loki-dashboard.json similarity index 100% rename from charts/cluster-addons/grafana-dashboards/loki-dashboard.json rename to charts/openstack-cluster/grafana-dashboards/loki-dashboard.json diff --git a/charts/cluster-addons/grafana-dashboards/nvidia-dcgm-exporter-dashboard_rev2.json b/charts/openstack-cluster/grafana-dashboards/nvidia-dcgm-exporter-dashboard_rev2.json similarity index 100% rename from charts/cluster-addons/grafana-dashboards/nvidia-dcgm-exporter-dashboard_rev2.json rename to charts/openstack-cluster/grafana-dashboards/nvidia-dcgm-exporter-dashboard_rev2.json diff --git a/charts/openstack-cluster/templates/_helpers.tpl b/charts/openstack-cluster/templates/_helpers.tpl index a98390c..ca3481f 100644 --- a/charts/openstack-cluster/templates/_helpers.tpl +++ b/charts/openstack-cluster/templates/_helpers.tpl @@ -28,47 +28,39 @@ Common labels helm.sh/chart: {{ include "openstack-cluster.chart" . }} capi.stackhpc.com/managed-by: {{ .Release.Service }} capi.stackhpc.com/infrastructure-provider: openstack +{{- end -}} + +{{/* +Selector labels for cluster-level resources +*/}} +{{- define "openstack-cluster.selectorLabels" -}} capi.stackhpc.com/cluster: {{ include "openstack-cluster.clusterName" . }} {{- end -}} {{/* -Component labels +Labels for cluster-level resources */}} -{{- define "openstack-cluster.componentLabels" -}} +{{- define "openstack-cluster.labels" -}} +{{ include "openstack-cluster.commonLabels" . }} +{{ include "openstack-cluster.selectorLabels" . }} +{{- end -}} + +{{/* +Selector labels for component-level resources +*/}} +{{- define "openstack-cluster.componentSelectorLabels" -}} {{- $ctx := index . 0 -}} {{- $componentName := index . 1 -}} -{{- include "openstack-cluster.commonLabels" $ctx }} +{{ include "openstack-cluster.selectorLabels" $ctx }} capi.stackhpc.com/component: {{ $componentName }} {{- end -}} {{/* -Control plane selector labels +Labels for component-level resources */}} -{{- define "openstack-cluster.controlPlaneSelectorLabels" -}} -capi.stackhpc.com/cluster: {{ include "openstack-cluster.clusterName" . }} -capi.stackhpc.com/component: control-plane -{{- end -}} - -{{/* -Node group labels -*/}} -{{- define "openstack-cluster.nodeGroupLabels" -}} -{{- $ctx := index . 0 -}} -{{- $nodeGroupName := index . 1 -}} -{{- include "openstack-cluster.commonLabels" $ctx }} -capi.stackhpc.com/component: worker -capi.stackhpc.com/node-group: {{ $nodeGroupName }} -{{- end -}} - -{{/* -Node group selector labels -*/}} -{{- define "openstack-cluster.nodeGroupSelectorLabels" -}} -{{- $ctx := index . 0 -}} -{{- $nodeGroupName := index . 1 -}} -capi.stackhpc.com/cluster: {{ include "openstack-cluster.clusterName" $ctx }} -capi.stackhpc.com/component: worker -capi.stackhpc.com/node-group: {{ $nodeGroupName }} +{{- define "openstack-cluster.componentLabels" -}} +{{ include "openstack-cluster.commonLabels" (index . 0) }} +{{ include "openstack-cluster.componentSelectorLabels" . }} {{- end -}} {{/* diff --git a/charts/openstack-cluster/templates/addons/_hook.tpl.yaml b/charts/openstack-cluster/templates/addons/_hook.tpl.yaml new file mode 100644 index 0000000..e08e01b --- /dev/null +++ b/charts/openstack-cluster/templates/addons/_hook.tpl.yaml @@ -0,0 +1,69 @@ +{{- define "openstack-cluster.hookJob" -}} +{{- $ctx := index . 0 }} +{{- $hook := index . 1 }} +{{- $componentName := index . 2 }} +{{- $scriptTemplate := index . 3 }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "openstack-cluster.componentName" (list $ctx $componentName) }} + labels: {{ include "openstack-cluster.componentLabels" (list $ctx $componentName) | nindent 4 }} + annotations: + helm.sh/hook: {{ $hook }} + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: {{ $ctx.Values.addons.hooks.backoffLimit }} + activeDeadlineSeconds: {{ $ctx.Values.addons.hooks.activeDeadlineSeconds }} + template: + metadata: + labels: {{ include "openstack-cluster.componentSelectorLabels" (list $ctx $componentName) | nindent 8 }} + spec: + {{- with $ctx.Values.addons.hooks.imagePullSecrets }} + imagePullSecrets: {{ toYaml . | nindent 8 }} + {{- end }} + securityContext: {{ toYaml $ctx.Values.addons.hooks.podSecurityContext | nindent 8 }} + restartPolicy: OnFailure + containers: + - name: {{ $componentName }} + image: {{ + printf "%s:%s" + $ctx.Values.addons.hooks.image.repository + (default $ctx.Chart.AppVersion $ctx.Values.addons.hooks.image.tag) + }} + imagePullPolicy: {{ $ctx.Values.addons.hooks.image.pullPolicy }} + securityContext: {{ toYaml $ctx.Values.addons.hooks.securityContext | nindent 12 }} + args: + - /bin/bash + - -c + - | + set -ex + test -f "$KUBECONFIG" || exit 0 + kubectl version || exit 0 + {{- include $scriptTemplate $ctx | nindent 16 }} + env: + - name: KUBECONFIG + value: /etc/kubernetes/config + resources: {{ toYaml $ctx.Values.addons.hooks.resources | nindent 12 }} + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + readOnly: true + hostNetwork: {{ $ctx.Values.addons.hooks.hostNetwork }} + {{- with $ctx.Values.addons.hooks.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with $ctx.Values.addons.hooks.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with $ctx.Values.addons.hooks.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: etc-kubernetes + secret: + secretName: {{ include "openstack-cluster.componentName" (list $ctx "kubeconfig") }} + optional: true + items: + - key: value + path: config +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/cni/calico.yaml b/charts/openstack-cluster/templates/addons/cni/calico.yaml new file mode 100644 index 0000000..ee71aa3 --- /dev/null +++ b/charts/openstack-cluster/templates/addons/cni/calico.yaml @@ -0,0 +1,61 @@ +##### +# Use a pre-upgrade hook to make sure the installation is annotated as belonging to Helm +##### +{{- define "openstack-cluster.cni-calico.hookScript" -}} +helm-adopt cni-calico {{ .Values.addons.cni.calico.release.namespace }} installation/default +{{- end }} + +{{- + if and + .Values.addons.enabled + .Values.addons.cni.enabled + (eq .Values.addons.cni.type "calico") +}} +--- +{{- + include + "openstack-cluster.hookJob" + (list . "pre-upgrade" "cni-calico-migrate" "openstack-cluster.cni-calico.hookScript") +}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "cni-calico") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "cni-calico") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + defaults: | + installation: + calicoNetwork: + bgp: Disabled + nodeAddressAutodetectionV4: + kubernetes: NodeInternalIP + ipPools: + {% for cidr in cluster.spec.clusterNetwork.pods.cidrBlocks %} + - cidr: {{ "{{" }} cidr {{ "}}" }} + encapsulation: VXLAN + {% endfor %} + overrides: | + {{- toYaml .Values.addons.cni.calico.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "cni-calico") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "cni-calico") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.cni.calico.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.cni.calico.release.namespace }} + releaseName: cni-calico + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "cni-calico") }}-config + key: defaults + - secret: + name: {{ include "openstack-cluster.componentName" (list . "cni-calico") }}-config + key: overrides +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/cni/cilium.yaml b/charts/openstack-cluster/templates/addons/cni/cilium.yaml new file mode 100644 index 0000000..0e3d688 --- /dev/null +++ b/charts/openstack-cluster/templates/addons/cni/cilium.yaml @@ -0,0 +1,40 @@ +{{- + if and + .Values.addons.enabled + .Values.addons.cni.enabled + (eq .Values.addons.cni.type "cilium") +}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "cni-cilium") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "cni-cilium") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + defaults: | + ipam: + mode: kubernetes + overrides: | + {{- toYaml .Values.addons.cni.cilium.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "cni-cilium") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "cni-cilium") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.cni.cilium.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.cni.cilium.release.namespace }} + releaseName: cni-cilium + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "cni-cilium") }}-config + key: defaults + - secret: + name: {{ include "openstack-cluster.componentName" (list . "cni-cilium") }}-config + key: overrides +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/ingress-nginx.yaml b/charts/openstack-cluster/templates/addons/ingress-nginx.yaml new file mode 100644 index 0000000..4e08945 --- /dev/null +++ b/charts/openstack-cluster/templates/addons/ingress-nginx.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.addons.enabled .Values.addons.ingress.nginx.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "ingress-nginx") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "ingress-nginx") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + values: | + {{- toYaml .Values.addons.ingress.nginx.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "ingress-nginx") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "ingress-nginx") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.ingress.nginx.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.ingress.nginx.release.namespace }} + releaseName: ingress-nginx + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "ingress-nginx") }}-config + key: values +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/kubernetes-dashboard.yaml b/charts/openstack-cluster/templates/addons/kubernetes-dashboard.yaml new file mode 100644 index 0000000..a138595 --- /dev/null +++ b/charts/openstack-cluster/templates/addons/kubernetes-dashboard.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.addons.enabled .Values.addons.kubernetesDashboard.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "kubernetes-dashboard") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "kubernetes-dashboard") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + # Enable the metrics scraper by default + defaults: | + metricsScraper: + enabled: true + overrides: | + {{- toYaml .Values.addons.kubernetesDashboard.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "kubernetes-dashboard") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "kubernetes-dashboard") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.kubernetesDashboard.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.kubernetesDashboard.release.namespace }} + releaseName: kubernetes-dashboard + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "kubernetes-dashboard") }}-config + key: defaults + - secret: + name: {{ include "openstack-cluster.componentName" (list . "kubernetes-dashboard") }}-config + key: overrides +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/mellanox-network-operator.yaml b/charts/openstack-cluster/templates/addons/mellanox-network-operator.yaml new file mode 100644 index 0000000..fdc7c7a --- /dev/null +++ b/charts/openstack-cluster/templates/addons/mellanox-network-operator.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.addons.enabled .Values.addons.mellanoxNetworkOperator.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "mellanox-network-operator") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "mellanox-network-operator") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + defaults: | + # Use the shared NFD + nfd: + enabled: false + # Deploy the default NICClusterPolicy + deployCR: true + # Deploy the OFED driver onto nodes with a suitable NIC + ofedDriver: + deploy: true + # OFED takes ages to deploy on low-resource nodes + # The startup probe has a fixed failure threshold of 60 + # So in order to give the drivers up to one hour to install, we use a period + # of 60 seconds for the startup probe + startupProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + # Deploy the RDMA shared device plugin to allow pods to access the RDMA device + rdmaSharedDevicePlugin: + deploy: true + # Disable all other features for now + sriovNetworkOperator: + enabled: false + sriovDevicePlugin: + deploy: false + secondaryNetwork: + deploy: false + overrides: | + {{- toYaml .Values.addons.mellanoxNetworkOperator.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "mellanox-network-operator") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "mellanox-network-operator") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.mellanoxNetworkOperator.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.mellanoxNetworkOperator.release.namespace }} + releaseName: mellanox-network-operator + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "mellanox-network-operator") }}-config + key: defaults + - secret: + name: {{ include "openstack-cluster.componentName" (list . "mellanox-network-operator") }}-config + key: overrides +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/metrics-server.yaml b/charts/openstack-cluster/templates/addons/metrics-server.yaml new file mode 100644 index 0000000..db05fb7 --- /dev/null +++ b/charts/openstack-cluster/templates/addons/metrics-server.yaml @@ -0,0 +1,49 @@ +##### +# Use a pre-upgrade hook to remove the old kustomize release +##### +{{- define "openstack-cluster.metrics-server.hookScript" -}} +helm status -n kustomize-releases metrics-server || exit 0 +helm delete -n kustomize-releases metrics-server +{{- end }} + +{{- if and .Values.addons.enabled .Values.addons.metricsServer.enabled }} +--- +{{- + include + "openstack-cluster.hookJob" + (list . "pre-upgrade" "metrics-server-migrate" "openstack-cluster.metrics-server.hookScript") +}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "metrics-server") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "metrics-server") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + defaults: | + args: + - --kubelet-insecure-tls + overrides: | + {{- toYaml .Values.addons.metricsServer.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "metrics-server") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "metrics-server") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.metricsServer.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.metricsServer.release.namespace }} + releaseName: metrics-server + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "metrics-server") }}-config + key: defaults + - secret: + name: {{ include "openstack-cluster.componentName" (list . "metrics-server") }}-config + key: overrides +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/monitoring/kube-prometheus-stack.yaml b/charts/openstack-cluster/templates/addons/monitoring/kube-prometheus-stack.yaml new file mode 100644 index 0000000..806eea7 --- /dev/null +++ b/charts/openstack-cluster/templates/addons/monitoring/kube-prometheus-stack.yaml @@ -0,0 +1,79 @@ +{{- define "openstack-cluster.kube-prometheus-stack.hookScript" -}} +# Ensure that the dashboard configmaps belong to the Helm release for the manifest object +helm-adopt \ + kube-prometheus-stack-dashboards \ + {{ .Values.addons.monitoring.kubePrometheusStack.release.namespace }} \ + configmap/nvidia-dcgm-exporter-dashboard \ + --namespace {{ .Values.addons.monitoring.kubePrometheusStack.release.namespace }} +# With the version bump to 40.x, kube-prometheus-stack picks up prometheus-node-exporter 4.x +# This changes the selector labels on the daemonset, which is an immutable field, so we remove +# the daemonset with the old labels before upgrading +# NOTE: Once the upgrade to 40.x has occurred, this will be a no-op +# https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#from-39x-to-40x +kubectl delete daemonset \ + -l release=kube-prometheus-stack,app=prometheus-node-exporter \ + -n {{ .Values.addons.monitoring.kubePrometheusStack.release.namespace }} +{{- end }} + +{{- if and .Values.addons.enabled .Values.addons.monitoring.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "kube-prometheus-stack") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "kube-prometheus-stack") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + values: | + {{- toYaml .Values.addons.monitoring.kubePrometheusStack.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "kube-prometheus-stack") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "kube-prometheus-stack") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.monitoring.kubePrometheusStack.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.monitoring.kubePrometheusStack.release.namespace }} + releaseName: kube-prometheus-stack + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "kube-prometheus-stack") }}-config + key: values +--- +{{- + include + "openstack-cluster.hookJob" + (list + . + "pre-upgrade" + "kube-prometheus-stack-migrate" + "openstack-cluster.kube-prometheus-stack.hookScript" + ) +}} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: Manifests +metadata: + name: {{ include "openstack-cluster.componentName" (list . "kube-prometheus-stack") }}-dashboards + labels: {{ include "openstack-cluster.componentLabels" (list . "kube-prometheus-stack") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + targetNamespace: {{ .Values.addons.monitoring.kubePrometheusStack.release.namespace }} + releaseName: kube-prometheus-stack-dashboards + manifestSources: + - template: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: nvidia-dcgm-exporter-dashboard + labels: + grafana_dashboard: "1" + data: + nvidia-dcgm-exporter-dashboard.json: | + {{- .Files.Get "grafana-dashboards/nvidia-dcgm-exporter-dashboard_rev2.json" | nindent 12 }} +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/monitoring/loki-stack.yaml b/charts/openstack-cluster/templates/addons/monitoring/loki-stack.yaml new file mode 100644 index 0000000..413ff9d --- /dev/null +++ b/charts/openstack-cluster/templates/addons/monitoring/loki-stack.yaml @@ -0,0 +1,103 @@ +{{- define "openstack-cluster.loki-stack.hookScript" -}} +# Ensure that the dashboard configmaps belong to the Helm release for the manifest object +helm-adopt \ + loki-stack-dashboards \ + {{ .Values.addons.monitoring.kubePrometheusStack.release.namespace }} \ + configmap/loki-stack-grafana-datasource \ + --namespace {{ .Values.addons.monitoring.kubePrometheusStack.release.namespace }} +helm-adopt \ + loki-stack-dashboards \ + {{ .Values.addons.monitoring.kubePrometheusStack.release.namespace }} \ + configmap/loki-stack-grafana-dashboard \ + --namespace {{ .Values.addons.monitoring.kubePrometheusStack.release.namespace }} +# At some point in the evolution of the promtail chart, the selector labels for the daemonset changed +# This is an immutable field, so we remove the daemonset with the old labels before upgrading +# NOTE: Once the upgrade to the new labels has occurred, this will be a no-op +kubectl delete daemonset \ + -l release=loki-stack,app=promtail \ + -n {{ .Values.addons.monitoring.lokiStack.release.namespace }} +{{- end }} + +{{- + if and + .Values.addons.enabled + .Values.addons.monitoring.enabled + .Values.addons.monitoring.lokiStack.enabled +}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "loki-stack") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "loki-stack") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + values: | + {{- toYaml .Values.addons.monitoring.lokiStack.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "loki-stack") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "loki-stack") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.monitoring.lokiStack.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.monitoring.lokiStack.release.namespace }} + releaseName: loki-stack + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "loki-stack") }}-config + key: values +--- +{{- + include + "openstack-cluster.hookJob" + (list + . + "pre-upgrade" + "loki-stack-migrate" + "openstack-cluster.loki-stack.hookScript" + ) +}} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: Manifests +metadata: + name: {{ include "openstack-cluster.componentName" (list . "loki-stack") }}-dashboards + labels: {{ include "openstack-cluster.componentLabels" (list . "loki-stack") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + targetNamespace: {{ .Values.addons.monitoring.kubePrometheusStack.release.namespace }} + releaseName: loki-stack-dashboards + manifestSources: + - template: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: loki-stack-grafana-datasource + labels: + grafana_datasource: "1" + data: + loki-datasource.yaml: |- + apiVersion: 1 + datasources: + - name: Loki + type: loki + url: http://loki-stack.{{ .Values.addons.monitoring.lokiStack.release.namespace }}:3100 + access: proxy + version: 1 + - template: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: loki-stack-grafana-dashboard + labels: + grafana_dashboard: "1" + data: + loki-dashboard.json: | + {{- .Files.Get "grafana-dashboards/loki-dashboard.json" | nindent 12 }} +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/nfd.yaml b/charts/openstack-cluster/templates/addons/nfd.yaml new file mode 100644 index 0000000..eac3730 --- /dev/null +++ b/charts/openstack-cluster/templates/addons/nfd.yaml @@ -0,0 +1,65 @@ +{{- + if and + .Values.addons.enabled + (or + .Values.addons.nodeFeatureDiscovery.enabled + .Values.addons.nvidiaGPUOperator.enabled + .Values.addons.mellanoxNetworkOperator.enabled + ) +}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "node-feature-discovery") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "node-feature-discovery") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + defaults: | + master: + extraLabelNs: + - nvidia.com + worker: + # Allow the NFD pods to be scheduled on all pods + tolerations: + - effect: "NoSchedule" + operator: "Exists" + # We want to be able to identify nodes with high-performance hardware + # So the whitelisted device classes are: + # 02 - Network Controllers (e.g. Ethernet, Infiniband) + # 03 - Display Controllers (e.g. GPUs) + # 0b40 - Co-processors + # 12 - Processing Accelerators (e.g. specialised AI inference chips) + config: + sources: + pci: + deviceClassWhitelist: + - "02" + - "03" + - "0b40" + - "12" + deviceLabelFields: + - vendor + overrides: | + {{- toYaml .Values.addons.nodeFeatureDiscovery.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "node-feature-discovery") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "node-feature-discovery") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.nodeFeatureDiscovery.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.nodeFeatureDiscovery.release.namespace }} + releaseName: node-feature-discovery + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "node-feature-discovery") }}-config + key: defaults + - secret: + name: {{ include "openstack-cluster.componentName" (list . "node-feature-discovery") }}-config + key: overrides +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/nvidia-gpu-operator.yaml b/charts/openstack-cluster/templates/addons/nvidia-gpu-operator.yaml new file mode 100644 index 0000000..547f984 --- /dev/null +++ b/charts/openstack-cluster/templates/addons/nvidia-gpu-operator.yaml @@ -0,0 +1,51 @@ +{{- if and .Values.addons.enabled .Values.addons.nvidiaGPUOperator.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "nvidia-gpu-operator") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "nvidia-gpu-operator") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + defaults: | + # Use the shared NFD + nfd: + enabled: false + # Export operator and node metrics in a Prometheus format. + # The component provides information on the status of the + # operator (e.g. reconciliation status, number of GPU enabled nodes). + nodeStatusExporter: + enabled: true + toolkit: + # Allowing the toolkit to edit /etc/containerd/config.toml (the default) + # breaks nvidia pod deployment on clusters with Harbor cache enabled. + # Instead make a new config file specifically for nvidia runtime config, + # which is parsed as an "include" in the main containerd config file. + # + # https://github.com/NVIDIA/gpu-operator/issues/301 + env: + - name: "CONTAINERD_CONFIG" + value: "/etc/containerd/conf.d/nvidia.toml" + overrides: | + {{- toYaml .Values.addons.nvidiaGPUOperator.release.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "nvidia-gpu-operator") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "nvidia-gpu-operator") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.nvidiaGPUOperator.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.nvidiaGPUOperator.release.namespace }} + releaseName: nvidia-gpu-operator + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "nvidia-gpu-operator") }}-config + key: defaults + - secret: + name: {{ include "openstack-cluster.componentName" (list . "nvidia-gpu-operator") }}-config + key: overrides +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/openstack/ccm.yaml b/charts/openstack-cluster/templates/addons/openstack/ccm.yaml new file mode 100644 index 0000000..41f4da2 --- /dev/null +++ b/charts/openstack-cluster/templates/addons/openstack/ccm.yaml @@ -0,0 +1,66 @@ +##### +# Use a pre-upgrade hook to remove the old kustomize release +##### +{{- define "openstack-cluster.ccm-openstack.hookScript" -}} +helm status -n kustomize-releases ccm-openstack || exit 0 +helm delete -n kustomize-releases ccm-openstack +{{- end }} + +{{- + if and + .Values.addons.enabled + .Values.addons.openstack.enabled + .Values.addons.openstack.ccm.enabled +}} +--- +{{- + include + "openstack-cluster.hookJob" + (list . "pre-upgrade" "ccm-openstack-migrate" "openstack-cluster.ccm-openstack.hookScript") +}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "ccm-openstack") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "ccm-openstack") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + defaults: | + secret: + create: false + cluster: + name: {{ include "openstack-cluster.clusterName" . }} + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + overrides: | + {{- toYaml .Values.addons.openstack.ccm.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "ccm-openstack") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "ccm-openstack") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.openstack.ccm.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.openstack.targetNamespace }} + releaseName: ccm-openstack + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "ccm-openstack") }}-config + key: defaults + - secret: + name: {{ include "openstack-cluster.componentName" (list . "ccm-openstack") }}-config + key: overrides +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/openstack/cloud-config.yaml b/charts/openstack-cluster/templates/addons/openstack/cloud-config.yaml new file mode 100644 index 0000000..9e81afd --- /dev/null +++ b/charts/openstack-cluster/templates/addons/openstack/cloud-config.yaml @@ -0,0 +1,64 @@ +{{- if and .Values.addons.enabled .Values.addons.openstack.enabled }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: Manifests +metadata: + name: {{ include "openstack-cluster.componentName" (list . "cloud-config") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "cloud-config") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + targetNamespace: {{ .Values.addons.openstack.targetNamespace }} + releaseName: cloud-config + manifestSources: + - template: | + apiVersion: v1 + kind: Secret + metadata: + name: cloud-config + data: + {{ "{{" }} cloud_identity.data | toyaml | indent(2) {{ "}}" }} + stringData: + cloud.conf: | + [Global] + use-clouds=true + clouds-file=/etc/config/clouds.yaml + cloud={{ .Values.cloudName }} + {%- if "cacert" in cloud_identity.data %} + ca-file=/etc/config/cacert + {%- else %} + tls-insecure=true + {%- endif %} + [Networking] + {{- $networkingItems := default dict .Values.addons.openstack.cloudConfig.Networking }} + {{- if hasKey $networkingItems "internal-network-name" }} + internal-network-name={{ index $networkingItems "internal-network-name" }} + {{- else }} + internal-network-name={{ "{{" }} infra_cluster.status.network.name {{ "}}" }} + {{- end }} + {{- range $netName, $netValue := omit $networkingItems "internal-network-name" }} + {{ $netName }}={{ $netValue }} + {{- end }} + [LoadBalancer] + {{- $lbItems := default dict .Values.addons.openstack.cloudConfig.LoadBalancer }} + {{- if hasKey $lbItems "floating-network-id" }} + floating-network-id={{ index $lbItems "floating-network-id" }} + {{- else }} + floating-network-id={{ "{{" }} infra_cluster.status.externalNetwork.id {{ "}}" }} + {{- end }} + {{- range $lbName, $lbValue := omit $lbItems "floating-network-id" }} + {{ $lbName }}={{ $lbValue }} + {{- end }} + {{- + range $section, $items := omit + .Values.addons.openstack.cloudConfig + "Global" + "LoadBalancer" + "Networking" + }} + [{{ $section }}] + {{- range $name, $value := $items }} + {{ $name }}={{ $value }} + {{- end }} + {{ end }} +{{- end }} diff --git a/charts/openstack-cluster/templates/addons/openstack/csi-cinder.yaml b/charts/openstack-cluster/templates/addons/openstack/csi-cinder.yaml new file mode 100644 index 0000000..c871629 --- /dev/null +++ b/charts/openstack-cluster/templates/addons/openstack/csi-cinder.yaml @@ -0,0 +1,116 @@ +# Use a pre-upgrade hook to move the Helm release into the new namespace +{{- define "openstack-cluster.csi-cinder.preUpgradeScript" -}} +{{- $targetNamespace := .Values.addons.openstack.targetNamespace }} +helm status -n kustomize-releases csi-cinder || exit 0 +helm-move csi-cinder kustomize-releases {{ $targetNamespace }} +{{- with .Values.addons.openstack.csiCinder.storageClass }} +{{- if .enabled }} +helm-adopt csi-cinder-storageclass {{ $targetNamespace }} storageclass/{{ .name }} +kubectl annotate storageclass/{{ .name }} "helm.sh/resource-policy=keep" +{{- end }} +{{- end }} +{{- end }} + +{{- + if and + .Values.addons.enabled + .Values.addons.openstack.enabled + .Values.addons.openstack.csiCinder.enabled +}} +--- +{{- + include + "openstack-cluster.hookJob" + (list . "pre-upgrade" "csi-cinder-migrate" "openstack-cluster.csi-cinder.preUpgradeScript") +}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openstack-cluster.componentName" (list . "csi-cinder") }}-config + labels: + {{- include "openstack-cluster.componentLabels" (list . "csi-cinder") | nindent 4 }} + {{ .Values.addons.watchLabel }}: "" +stringData: + # By default, we disable the storage class deployed by the cinder-csi chart + # We deploy our own instead as it gives us more control over the parameters + defaults: | + secret: + enabled: true + create: false + name: cloud-config + csi: + plugin: + # This has to be non-empty or the chart fails to render + volumes: + - name: cacert + emptyDir: {} + volumeMounts: + - name: cloud-config + mountPath: /etc/config + readOnly: true + - name: cloud-config + mountPath: /etc/kubernetes + readOnly: true + storageClass: + enabled: false + clusterID: {{ include "openstack-cluster.clusterName" . }} + overrides: | + {{- toYaml .Values.addons.openstack.csiCinder.values | nindent 4 }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: HelmRelease +metadata: + name: {{ include "openstack-cluster.componentName" (list . "csi-cinder") }} + labels: {{ include "openstack-cluster.componentLabels" (list . "csi-cinder") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + chart: {{ toYaml .Values.addons.openstack.csiCinder.chart | nindent 4 }} + targetNamespace: {{ .Values.addons.openstack.targetNamespace }} + releaseName: csi-cinder + valuesSources: + - secret: + name: {{ include "openstack-cluster.componentName" (list . "csi-cinder") }}-config + key: defaults + - secret: + name: {{ include "openstack-cluster.componentName" (list . "csi-cinder") }}-config + key: overrides +{{- if .Values.addons.openstack.csiCinder.storageClass.enabled }} +--- +apiVersion: addons.stackhpc.com/v1alpha1 +kind: Manifests +metadata: + name: {{ include "openstack-cluster.componentName" (list . "csi-cinder") }}-storageclass + labels: {{ include "openstack-cluster.componentLabels" (list . "csi-cinder") | nindent 4 }} +spec: + clusterName: {{ include "openstack-cluster.clusterName" . }} + bootstrap: true + targetNamespace: {{ .Values.addons.openstack.targetNamespace }} + releaseName: csi-cinder-storageclass + manifestSources: + - template: | + {{- with .Values.addons.openstack.csiCinder.storageClass }} + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: {{ .name }} + {{- if .isDefault }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" + {{- end }} + provisioner: cinder.csi.openstack.org + parameters: + availability: {{ .availabilityZone }} + {{- with .volumeType }} + type: {{ . }} + {{- end }} + reclaimPolicy: {{ .reclaimPolicy }} + allowVolumeExpansion: {{ .allowVolumeExpansion }} + volumeBindingMode: WaitForFirstConsumer + {{- with .allowedTopologies }} + allowedTopologies: {{ toYaml . | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/openstack-cluster/templates/cluster-openstack.yaml b/charts/openstack-cluster/templates/cluster-openstack.yaml index 78ab688..d8f7659 100644 --- a/charts/openstack-cluster/templates/cluster-openstack.yaml +++ b/charts/openstack-cluster/templates/cluster-openstack.yaml @@ -3,7 +3,7 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 kind: OpenStackCluster metadata: name: {{ include "openstack-cluster.clusterName" . }} - labels: {{ include "openstack-cluster.commonLabels" . | nindent 4 }} + labels: {{ include "openstack-cluster.labels" . | nindent 4 }} annotations: # We let Cluster API clean up the cluster resources # Deleting them ourselves, which CAPI is not expecting, can cause some nasty race conditions diff --git a/charts/openstack-cluster/templates/cluster.yaml b/charts/openstack-cluster/templates/cluster.yaml index 98acc82..4c139c0 100644 --- a/charts/openstack-cluster/templates/cluster.yaml +++ b/charts/openstack-cluster/templates/cluster.yaml @@ -3,7 +3,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: {{ include "openstack-cluster.clusterName" . }} - labels: {{ include "openstack-cluster.commonLabels" . | nindent 4 }} + labels: {{ include "openstack-cluster.labels" . | nindent 4 }} spec: clusterNetwork: {{ .Values.kubeNetwork | toYaml | nindent 4 }} controlPlaneRef: diff --git a/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml b/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml index 522ff9e..fab21b7 100644 --- a/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml +++ b/charts/openstack-cluster/templates/control-plane/kubeadm-control-plane.yaml @@ -80,7 +80,7 @@ spec: rolloutStrategy: {{ toYaml .Values.controlPlane.rolloutStrategy | nindent 4 }} machineTemplate: metadata: - labels: {{ include "openstack-cluster.controlPlaneSelectorLabels" . | nindent 8 }} + labels: {{ include "openstack-cluster.componentSelectorLabels" (list . "control-plane") | nindent 8 }} infrastructureRef: kind: OpenStackMachineTemplate apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 diff --git a/charts/openstack-cluster/templates/control-plane/machine-health-check.yaml b/charts/openstack-cluster/templates/control-plane/machine-health-check.yaml index 3993377..aa1138a 100644 --- a/charts/openstack-cluster/templates/control-plane/machine-health-check.yaml +++ b/charts/openstack-cluster/templates/control-plane/machine-health-check.yaml @@ -8,6 +8,6 @@ metadata: spec: clusterName: {{ include "openstack-cluster.clusterName" . }} selector: - matchLabels: {{ include "openstack-cluster.controlPlaneSelectorLabels" . | nindent 6 }} + matchLabels: {{ include "openstack-cluster.componentSelectorLabels" (list . "control-plane") | nindent 6 }} {{- toYaml .Values.controlPlane.healthCheck.spec | nindent 2 }} {{- end }} diff --git a/charts/openstack-cluster/templates/node-group/kubeadm-config-template.yaml b/charts/openstack-cluster/templates/node-group/kubeadm-config-template.yaml index 176f6ec..f5da7de 100644 --- a/charts/openstack-cluster/templates/node-group/kubeadm-config-template.yaml +++ b/charts/openstack-cluster/templates/node-group/kubeadm-config-template.yaml @@ -35,7 +35,9 @@ apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: name: {{ include "openstack-cluster.nodegroup.kct.name" (list $ $nodeGroup) }} - labels: {{ include "openstack-cluster.nodeGroupLabels" (list $ $nodeGroup.name) | nindent 4 }} + labels: + {{- include "openstack-cluster.componentLabels" (list $ "worker") | nindent 4 }} + capi.stackhpc.com/node-group: {{ $nodeGroup.name }} annotations: capi.stackhpc.com/template-checksum: {{ include "openstack-cluster.nodegroup.kct.checksum" (list $ $nodeGroup) }} # We let Cluster API clean up the cluster resources diff --git a/charts/openstack-cluster/templates/node-group/machine-deployment.yaml b/charts/openstack-cluster/templates/node-group/machine-deployment.yaml index 895a46c..1ebbee7 100644 --- a/charts/openstack-cluster/templates/node-group/machine-deployment.yaml +++ b/charts/openstack-cluster/templates/node-group/machine-deployment.yaml @@ -5,7 +5,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: name: {{ include "openstack-cluster.componentName" (list $ $nodeGroup.name) }} - labels: {{ include "openstack-cluster.nodeGroupLabels" (list $ $nodeGroup.name) | nindent 4 }} + labels: + {{- include "openstack-cluster.componentLabels" (list $ "worker") | nindent 4 }} + capi.stackhpc.com/node-group: {{ $nodeGroup.name }} {{- if $nodeGroup.autoscale }} {{- $machineCountMin := $nodeGroup.machineCountMin | @@ -32,10 +34,14 @@ spec: {{- end }} strategy: {{ toYaml $nodeGroup.rolloutStrategy | nindent 4 }} selector: - matchLabels: {{ include "openstack-cluster.nodeGroupSelectorLabels" (list $ $nodeGroup.name) | nindent 6 }} + matchLabels: + {{- include "openstack-cluster.componentSelectorLabels" (list $ "worker") | nindent 6 }} + capi.stackhpc.com/node-group: {{ $nodeGroup.name }} template: metadata: - labels: {{ include "openstack-cluster.nodeGroupSelectorLabels" (list $ $nodeGroup.name) | nindent 8 }} + labels: + {{- include "openstack-cluster.componentSelectorLabels" (list $ "worker") | nindent 8 }} + capi.stackhpc.com/node-group: {{ $nodeGroup.name }} spec: clusterName: {{ include "openstack-cluster.clusterName" $ }} version: {{ $.Values.global.kubernetesVersion }} diff --git a/charts/openstack-cluster/templates/node-group/machine-health-check.yaml b/charts/openstack-cluster/templates/node-group/machine-health-check.yaml index cee5fc6..4d9ac4d 100644 --- a/charts/openstack-cluster/templates/node-group/machine-health-check.yaml +++ b/charts/openstack-cluster/templates/node-group/machine-health-check.yaml @@ -6,11 +6,15 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: {{ include "openstack-cluster.componentName" (list $ $nodeGroup.name) }} - labels: {{ include "openstack-cluster.nodeGroupLabels" (list $ $nodeGroup.name) | nindent 4 }} + labels: + {{- include "openstack-cluster.componentLabels" (list $ "worker") | nindent 4 }} + capi.stackhpc.com/node-group: {{ $nodeGroup.name }} spec: clusterName: {{ include "openstack-cluster.clusterName" $ }} selector: - matchLabels: {{ include "openstack-cluster.nodeGroupSelectorLabels" (list $ $nodeGroup.name) | nindent 6 }} + matchLabels: + {{- include "openstack-cluster.componentSelectorLabels" (list $ "worker") | nindent 6 }} + capi.stackhpc.com/node-group: {{ $nodeGroup.name }} {{- toYaml $nodeGroup.healthCheck.spec | nindent 2 }} {{- end }} {{- end }} diff --git a/charts/openstack-cluster/templates/node-group/openstack-machine-template.yaml b/charts/openstack-cluster/templates/node-group/openstack-machine-template.yaml index bdd7209..a81bf41 100644 --- a/charts/openstack-cluster/templates/node-group/openstack-machine-template.yaml +++ b/charts/openstack-cluster/templates/node-group/openstack-machine-template.yaml @@ -51,7 +51,9 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6 kind: OpenStackMachineTemplate metadata: name: {{ include "openstack-cluster.nodegroup.mt.name" (list $ $nodeGroup) }} - labels: {{ include "openstack-cluster.nodeGroupLabels" (list $ $nodeGroup.name) | nindent 4 }} + labels: + {{- include "openstack-cluster.componentLabels" (list $ "worker") | nindent 4 }} + capi.stackhpc.com/node-group: {{ $nodeGroup.name }} annotations: capi.stackhpc.com/template-checksum: {{ include "openstack-cluster.nodegroup.mt.checksum" (list $ $nodeGroup) }} # We let Cluster API clean up the cluster resources diff --git a/charts/openstack-cluster/values.yaml b/charts/openstack-cluster/values.yaml index 9693cab..102dac6 100644 --- a/charts/openstack-cluster/values.yaml +++ b/charts/openstack-cluster/values.yaml @@ -3,17 +3,19 @@ global: # The Kubernetes version of the cluster # This should match the version of kubelet and kubeadm in the image kubernetesVersion: - # The name of an existing secret containing a clouds.yaml and optional cacert - cloudCredentialsSecretName: +# The name of an existing secret containing a clouds.yaml and optional cacert +cloudCredentialsSecretName: +# OR # Content for the clouds.yaml file # Having this as a top-level item allows a clouds.yaml file from OpenStack to be used as a values file clouds: -# The name of the cloud to use from the specified clouds -cloudName: openstack # The PEM-encoded CA certificate for the specified cloud cloudCACert: +# The name of the cloud to use from the specified clouds.yaml +cloudName: openstack + # The name of the image to use for cluster machines # This is used when creating machines using ephemeral root disks machineImage: @@ -289,10 +291,10 @@ autoscaler: # These are the current latest versions for each Kubernetes minor version # If a tag is not specified here for the target Kubernetes version, vX.Y.0 is used tags: - "1.23": v1.23.0 - "1.22": v1.22.2 - "1.21": v1.21.2 - "1.20": v1.20.2 + "1.25": v1.25.0 + "1.24": v1.24.0 + "1.23": v1.23.1 + "1.22": v1.22.3 imagePullSecrets: [] # Pod-level security context podSecurityContext: @@ -317,36 +319,217 @@ autoscaler: addons: # Indicates if cluster addons should be deployed enabled: true - # Indicates to the cluster addons that they are being deployed as part of a Cluster API cluster - # This means that they will wait for the control plane to stabilise before installing, except - # for the bootstrap addons which just wait for it be become available - clusterApi: true - clusterName: "{{ include \"openstack-cluster.clusterName\" . }}" - # The Kubernetes version for the addons should be v. - kubernetesVersion: "v{{ .Values.global.kubernetesVersion | splitList \".\" | reverse | rest | reverse | join \".\" }}" - # Launch addons on the workload cluster using the kubeconfig file created by CAPI - kubeconfigSecret: - name: "{{ include \"openstack-cluster.componentName\" (list . \"kubeconfig\") }}" - key: value - # By default, enable the OpenStack integrations - openstack: + + # The label to use to indicate that a configmap or secret should be watched + watchLabel: addons.stackhpc.com/watch + + # Settings for hook jobs + hooks: + image: + repository: ghcr.io/stackhpc/k8s-utils + tag: # Defaults to chart appVersion if not given + pullPolicy: IfNotPresent + imagePullSecrets: [] + backoffLimit: 1000 + activeDeadlineSeconds: 3600 + podSecurityContext: + runAsNonRoot: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + resources: {} + hostNetwork: false + tolerations: [] + nodeSelector: {} + affinity: {} + + # Settings for the CNI addon + cni: + # Indicates if a CNI should be deployed enabled: true - cloudCredentialsSecretName: "{{ include \"openstack-cluster.cloudCredentialsSecretName\" . }}" - monitoring: - kubePrometheusStack: + # The CNI to deploy - supported values are calico or cilium + type: calico + # Settings for the calico CNI + # See https://projectcalico.docs.tigera.io/getting-started/kubernetes/helm + calico: + chart: + repo: https://projectcalico.docs.tigera.io/charts + name: tigera-operator + version: v3.23.3 release: - values: - # Use the metrics port rather than the client as it does not require certificate auth - kubeEtcd: - service: - port: 2381 - targetPort: 2381 - # Use the correct port for kube-scheduler - kubeScheduler: - service: - port: 10259 - targetPort: 10259 - serviceMonitor: - https: true - # The certificate is valid for 127.0.0.1 - insecureSkipVerify: true + namespace: tigera-operator + values: {} + # Settings for the Cilium CNI + # See https://docs.cilium.io/en/stable/gettingstarted/k8s-install-helm/ for details + cilium: + chart: + repo: https://helm.cilium.io/ + name: cilium + version: 1.11.1 + release: + namespace: kube-system + values: {} + + # Settings for the OpenStack integrations + openstack: + # Indicates if the OpenStack integrations should be enabled + enabled: true + # The target namespace for the OpenStack integrations + targetNamespace: openstack-system + # cloud-config options for the OpenStack integrations + # The [Global] section is configured to use the target cloud + # See https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md#config-openstack-cloud-controller-manager + # and https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md#block-storage + cloudConfig: + # By default, ignore volume AZs for Cinder as most clouds have a single globally-attachable Cinder AZ + BlockStorage: + ignore-volume-az: true + # Settings for the Cloud Controller Manager (CCM) + ccm: + # Indicates if the OpenStack CCM should be enabled + # By default, the CCM is enabled if the OpenStack integrations are enabled + # See https://github.com/kubernetes/cloud-provider-openstack/blob/master/charts/openstack-cloud-controller-manager/values.yaml + enabled: true + chart: + repo: https://kubernetes.github.io/cloud-provider-openstack + name: openstack-cloud-controller-manager + version: 1.3.0 + values: {} + # Settings for the Cinder CSI plugin + csiCinder: + # Indicates if the Cinder CSI should be enabled + # By default, it is enabled if the OpenStack integrations are enabled + # See https://github.com/kubernetes/cloud-provider-openstack/blob/master/charts/cinder-csi-plugin/values.yaml + enabled: true + chart: + repo: https://kubernetes.github.io/cloud-provider-openstack + name: openstack-cinder-csi + version: 2.2.0 + values: {} + # Variables affecting the definition of the storage class + storageClass: + # Indicates if the storage class should be enabled + enabled: true + # The name of the storage class + name: csi-cinder + # Indicates if the storage class should be annotated as the default storage class + isDefault: true + # The reclaim policy for the storage class + reclaimPolicy: Delete + # Indicates if volume expansion is allowed + allowVolumeExpansion: true + # The Cinder availability zone to use for volumes provisioned by the storage class + availabilityZone: nova + # The Cinder volume type to use for volumes provisioned by the storage class + # If not given, the default volume type will be used + volumeType: + # The allowed topologies for the storage class + allowedTopologies: + + # Settings for the metrics server + # https://github.com/kubernetes-sigs/metrics-server#helm-chart + metricsServer: + # Indicates if the metrics server should be deployed + enabled: true + chart: + repo: https://kubernetes-sigs.github.io/metrics-server + name: metrics-server + version: 3.8.2 + release: + namespace: kube-system + values: {} + + # Settings for the Kubernetes dashboard + # https://github.com/kubernetes/dashboard/tree/master/charts/helm-chart/kubernetes-dashboard + kubernetesDashboard: + # Indicates if the Kubernetes dashboard should be enabled + enabled: false + chart: + repo: https://kubernetes.github.io/dashboard + name: kubernetes-dashboard + version: 5.10.0 + release: + namespace: kubernetes-dashboard + values: {} + + # Settings for ingress controllers + ingress: + # Settings for the Nginx ingress controller + # https://github.com/kubernetes/ingress-nginx/tree/main/charts/ingress-nginx#configuration + nginx: + # Indicates if the Nginx ingress controller should be enabled + enabled: false + chart: + repo: https://kubernetes.github.io/ingress-nginx + name: ingress-nginx + version: 4.2.5 + release: + namespace: ingress-nginx + values: {} + + # Settings for cluster monitoring + monitoring: + # Indicates if the cluster monitoring should be enabled + enabled: false + kubePrometheusStack: + chart: + repo: https://prometheus-community.github.io/helm-charts + name: kube-prometheus-stack + version: 40.1.0 + release: + namespace: monitoring-system + values: {} + lokiStack: + enabled: true + chart: + repo: https://grafana.github.io/helm-charts + name: loki-stack + version: 2.8.2 + release: + namespace: monitoring-system + values: {} + + # Settings for node feature discovery + # https://github.com/kubernetes-sigs/node-feature-discovery/tree/master/deployment/helm/node-feature-discovery + nodeFeatureDiscovery: + # Indicates if node feature discovery should be enabled + enabled: true + chart: + repo: https://kubernetes-sigs.github.io/node-feature-discovery/charts + name: node-feature-discovery + version: 0.11.2 + release: + namespace: node-feature-discovery + values: {} + + # Settings for the NVIDIA GPU operator + nvidiaGPUOperator: + # Indicates if the NVIDIA GPU operator should be enabled + # Note that because it uses node feature discovery to run only on nodes + # with an NVIDIA GPU available, the overhead of enabling this on clusters + # that do not need it now but may need it in the future is low + enabled: true + chart: + repo: https://nvidia.github.io/gpu-operator + name: gpu-operator + version: v1.11.1 + release: + namespace: gpu-operator + values: {} + + # Settings for the Mellanox network operator + mellanoxNetworkOperator: + # Indicates if the network operator should be enabled + # Note that because it uses node feature discovery to run only on nodes + # with a Mellanox NIC available, the overhead of enabling this on clusters + # that do not need it now but may need it in the future is low + enabled: true + chart: + repo: https://mellanox.github.io/network-operator + name: network-operator + version: 1.3.0 + release: + namespace: network-operator + values: {} diff --git a/utils/Dockerfile b/utils/Dockerfile index b3b8640..521bcaf 100644 --- a/utils/Dockerfile +++ b/utils/Dockerfile @@ -16,48 +16,9 @@ RUN groupadd --gid $UTILS_GID $UTILS_GROUP && \ RUN apt-get update && \ apt-get install -y curl git jq python3 python3-pip tini && \ - rm -rf /var/lib/apt/lists/* && \ - pip install --no-cache ruamel.yaml + rm -rf /var/lib/apt/lists/* -COPY --from=hairyhenderson/gomplate:v3.10.0 /gomplate /usr/bin/gomplate - -ARG YQ_VN=v4.25.1 -RUN set -ex; \ - OS_ARCH="$(uname -m)"; \ - case "$OS_ARCH" in \ - x86_64) yq_arch=amd64 ;; \ - aarch64) yq_arch=arm64 ;; \ - *) false ;; \ - esac; \ - curl -fsSL https://github.com/mikefarah/yq/releases/download/${YQ_VN}/yq_linux_${yq_arch} -o /usr/bin/yq; \ - chmod +x /usr/bin/yq; \ - /usr/bin/yq --version - -ARG KUBECTL_VN_1_20=v1.20.15 -RUN set -ex; \ - OS_ARCH="$(uname -m)"; \ - case "$OS_ARCH" in \ - x86_64) kubectl_arch=amd64 ;; \ - aarch64) kubectl_arch=arm64 ;; \ - *) false ;; \ - esac; \ - curl -fsSL https://dl.k8s.io/release/${KUBECTL_VN_1_20}/bin/linux/${kubectl_arch}/kubectl -o /usr/bin/kubectl-v1.20; \ - chmod +x /usr/bin/kubectl-v1.20; \ - /usr/bin/kubectl-v1.20 version --client - -ARG KUBECTL_VN_1_21=v1.21.12 -RUN set -ex; \ - OS_ARCH="$(uname -m)"; \ - case "$OS_ARCH" in \ - x86_64) kubectl_arch=amd64 ;; \ - aarch64) kubectl_arch=arm64 ;; \ - *) false ;; \ - esac; \ - curl -fsSL https://dl.k8s.io/release/${KUBECTL_VN_1_21}/bin/linux/${kubectl_arch}/kubectl -o /usr/bin/kubectl-v1.21; \ - chmod +x /usr/bin/kubectl-v1.21; \ - /usr/bin/kubectl-v1.21 version --client - -ARG KUBECTL_VN_1_22=v1.22.9 +ARG KUBECTL_VN_1_22=v1.22.13 RUN set -ex; \ OS_ARCH="$(uname -m)"; \ case "$OS_ARCH" in \ @@ -69,7 +30,7 @@ RUN set -ex; \ chmod +x /usr/bin/kubectl-v1.22; \ /usr/bin/kubectl-v1.22 version --client -ARG KUBECTL_VN_1_23=v1.23.6 +ARG KUBECTL_VN_1_23=v1.23.10 RUN set -ex; \ OS_ARCH="$(uname -m)"; \ case "$OS_ARCH" in \ @@ -81,7 +42,7 @@ RUN set -ex; \ chmod +x /usr/bin/kubectl-v1.23; \ /usr/bin/kubectl-v1.23 version --client -ARG KUBECTL_VN_1_24=v1.24.0 +ARG KUBECTL_VN_1_24=v1.24.4 RUN set -ex; \ OS_ARCH="$(uname -m)"; \ case "$OS_ARCH" in \ @@ -93,10 +54,22 @@ RUN set -ex; \ chmod +x /usr/bin/kubectl-v1.24; \ /usr/bin/kubectl-v1.24 version --client +ARG KUBECTL_VN_1_25=v1.25.0 +RUN set -ex; \ + OS_ARCH="$(uname -m)"; \ + case "$OS_ARCH" in \ + x86_64) kubectl_arch=amd64 ;; \ + aarch64) kubectl_arch=arm64 ;; \ + *) false ;; \ + esac; \ + curl -fsSL https://dl.k8s.io/release/${KUBECTL_VN_1_25}/bin/linux/${kubectl_arch}/kubectl -o /usr/bin/kubectl-v1.25; \ + chmod +x /usr/bin/kubectl-v1.25; \ + /usr/bin/kubectl-v1.25 version --client + ENV HELM_CACHE_HOME /tmp/helm/cache ENV HELM_CONFIG_HOME /tmp/helm/config ENV HELM_DATA_HOME /tmp/helm/data -ARG HELM_VERSION=v3.8.2 +ARG HELM_VERSION=v3.9.4 RUN set -ex; \ OS_ARCH="$(uname -m)"; \ case "$OS_ARCH" in \ @@ -108,20 +81,7 @@ RUN set -ex; \ tar -xz --strip-components 1 -C /usr/bin linux-${helm_arch}/helm; \ helm version -ARG KUSTOMIZE_VERSION=v4.5.4 -RUN set -ex; \ - OS_ARCH="$(uname -m)"; \ - case "$OS_ARCH" in \ - x86_64) kustomize_arch=amd64 ;; \ - aarch64) kustomize_arch=arm64 ;; \ - *) false ;; \ - esac; \ - curl -fsSL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_${kustomize_arch}.tar.gz | \ - tar -xz -C /usr/bin; \ - chmod +x /usr/bin/kustomize; \ - kustomize version - -ENV KUBECTL_VN_LATEST v1.24 +ENV KUBECTL_VN_LATEST v1.25 COPY ./bin/* /usr/bin/ USER $UTILS_UID diff --git a/utils/bin/helm-adopt b/utils/bin/helm-adopt new file mode 100755 index 0000000..941fcfb --- /dev/null +++ b/utils/bin/helm-adopt @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +##### +# Script that adopts the specified resource into the specified release +##### + +set -eo pipefail + +RELEASE_NAME= +RELEASE_NAMESPACE= +OBJECT= +OBJECT_NAMESPACE= + +while :; do + case $1 in + --help) + echo "Adopts a resource into a Helm release." + echo "" + echo "Usage: helm-adopt RELEASE-NAME RELEASE-NAMESPACE OBJECT [--namespace OBJECT-NAMESPACE]" + exit + ;; + --release-namespace) + RELEASE_NAMESPACE="$2" + shift + ;; + --namespace) + OBJECT_NAMESPACE="$2" + shift + ;; + ?*) + if [ -z "$RELEASE_NAME" ]; then + RELEASE_NAME="$1" + elif [ -z "$RELEASE_NAMESPACE" ]; then + RELEASE_NAMESPACE="$1" + elif [ -z "$OBJECT" ]; then + OBJECT="$1" + else + echo "Too many arguments" >&2 + exit 1 + fi + ;; + *) + break + esac + shift +done + +if [ -z "$RELEASE_NAME" ]; then + echo "RELEASE-NAME was not given" >&2 + exit 1 +elif [ -z "$RELEASE_NAMESPACE" ]; then + echo "RELEASE-NAMESPACE was not given" >&2 + exit 1 +elif [ -z "$OBJECT" ]; then + echo "OBJECT was not given" >&2 + exit 1 +fi + +KUBECTL_ARGS="$OBJECT" +[ -n "$OBJECT_NAMESPACE" ] && KUBECTL_ARGS="$KUBECTL_ARGS --namespace $OBJECT_NAMESPACE" + +kubectl get $KUBECTL_ARGS || exit 0 +kubectl label --overwrite $KUBECTL_ARGS "app.kubernetes.io/managed-by=Helm" +kubectl annotate --overwrite $KUBECTL_ARGS "meta.helm.sh/release-name=$RELEASE_NAME" +kubectl annotate --overwrite $KUBECTL_ARGS "meta.helm.sh/release-namespace=$RELEASE_NAMESPACE" diff --git a/utils/bin/helm-delete b/utils/bin/helm-delete deleted file mode 100755 index 3a92f54..0000000 --- a/utils/bin/helm-delete +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -##### -# Wrapper around "helm delete" that considers the delete successful if -# the release does not exist -##### - -RELEASE=$1 -shift - -NAMESPACE_ARG= -HELM_ARGS= - -while :; do - case $1 in - --help) - helm delete --help - exit - ;; - --namespace) - NAMESPACE_ARG="$1 $2" - shift - ;; - ?*) - HELM_ARGS="$HELM_ARGS $1" - ;; - *) - break - esac - shift -done - -set -e - -if helm-exists $RELEASE $NAMESPACE_ARG; then - exec helm delete $RELEASE $NAMESPACE_ARG $HELM_ARGS -fi diff --git a/utils/bin/helm-exists b/utils/bin/helm-exists deleted file mode 100755 index b21630c..0000000 --- a/utils/bin/helm-exists +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -##### -# Wrapper around "helm status" that just exits cleanly if a release exists and -# exits with an error if not -##### - -RELEASE=$1 -shift - -NAMESPACE_ARG= -HELM_ARGS= - -while :; do - case $1 in - --namespace) - NAMESPACE_ARG="$1 $2" - shift - ;; - ?*) - HELM_ARGS="$HELM_ARGS $1" - ;; - *) - break - esac - shift -done - -set -e - -exec helm status $RELEASE $NAMESPACE_ARG > /dev/null 2>&1 diff --git a/utils/bin/helm-move b/utils/bin/helm-move new file mode 100755 index 0000000..7acae69 --- /dev/null +++ b/utils/bin/helm-move @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +##### +# Script that moves the specified Helm release (NOT resources!) from one namespace to another +##### + +set -eo pipefail + +RELEASE_NAME= +FROM_NAMESPACE= +TO_NAMESPACE= + +while :; do + case $1 in + --help) + echo "Moves the specified Helm release from one namespace to another." + echo "" + echo "WARNING: This script does NOT move resources, only the release itself." + echo " It should only be used with charts that explicitly specify resource namespaces." + echo "" + echo "Usage: helm-move RELEASE-NAME FROM-NAMESPACE TO-NAMESPACE" + exit + ;; + ?*) + if [ -z "$RELEASE_NAME" ]; then + RELEASE_NAME="$1" + elif [ -z "$FROM_NAMESPACE" ]; then + FROM_NAMESPACE="$1" + elif [ -z "$TO_NAMESPACE" ]; then + TO_NAMESPACE="$1" + else + echo "Too many arguments" >&2 + exit 1 + fi + ;; + *) + break + esac + shift +done + +if [ -z "$RELEASE_NAME" ]; then + echo "RELEASE-NAME was not given" >&2 + exit 1 +elif [ -z "$FROM_NAMESPACE" ]; then + echo "FROM-NAMESPACE was not given" >&2 + exit 1 +elif [ -z "$TO_NAMESPACE" ]; then + echo "TO-NAMESPACE was not given" >&2 + exit 1 +fi + + +# Make sure that the target namespace exists +kubectl create ns "$TO_NAMESPACE" || true + +# Move each secret that corresponds to a revision of the release to the new namespace +for secret in $(kubectl -n $FROM_NAMESPACE get secret -o name --field-selector "type=helm.sh/release.v1" -l "name=$RELEASE_NAME"); do + # We need to replace the namespace in the release data + release="$( + kubectl -n $FROM_NAMESPACE get $secret -o go-template='{{.data.release}}' | + base64 -d | + base64 -d | + gzip -d | + jq -c ".namespace=\"$TO_NAMESPACE\"" | + gzip | + base64 | + base64 + )" + # Copy the secret to a new namespace, modifying it as it goes + kubectl -n $FROM_NAMESPACE get $secret -o json | + jq -c 'del(.metadata.creationTimestamp)' | + jq -c 'del(.metadata.resourceVersion)' | + jq -c 'del(.metadata.uid)' | + jq -c ".metadata.namespace=\"$TO_NAMESPACE\"" | + jq -c ".data.release=\"$release\"" | + kubectl create -f - + # Remove the old secret + kubectl -n $FROM_NAMESPACE delete $secret +done diff --git a/utils/bin/helm-upgrade b/utils/bin/helm-upgrade deleted file mode 100755 index a860f65..0000000 --- a/utils/bin/helm-upgrade +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -##### -# Wrapper around "helm upgrade" that is able to rollback a failed release before retrying -##### - -RELEASE=$1 -CHART=$2 -shift 2 - -NAMESPACE_ARG= -TIMEOUT_ARG= -HELM_ARGS= - -while :; do - case $1 in - --help) - helm upgrade --help - exit - ;; - --namespace) - NAMESPACE_ARG="$1 $2" - shift - ;; - --timeout) - TIMEOUT_ARG="$1 $2" - shift - ;; - ?*) - HELM_ARGS="$HELM_ARGS $1" - ;; - *) - break - esac - shift -done - -set -eo pipefail - -if helm-exists $RELEASE $NAMESPACE_ARG; then - status="$(helm status $RELEASE $NAMESPACE_ARG --output json | jq -r '.info.status')" - if [ "$status" = "pending-install" ]; then - echo "Deleting failed release..." - helm delete $RELEASE $NAMESPACE_ARG --wait $TIMEOUT_ARG - elif [ "$status" = "pending-upgrade" ]; then - echo "Rolling back failed upgrade..." - helm rollback $RELEASE $NAMESPACE_ARG --cleanup-on-fail --wait --wait-for-jobs $TIMEOUT_ARG - elif [ "$status" = "pending-rollback" ]; then - echo "Completing pending rollback..." - helm rollback $RELEASE $NAMESPACE_ARG --cleanup-on-fail --wait --wait-for-jobs $TIMEOUT_ARG - fi -fi - -exec helm upgrade $RELEASE $CHART $NAMESPACE_ARG $TIMEOUT_ARG $HELM_ARGS diff --git a/utils/bin/make-chart b/utils/bin/make-chart deleted file mode 100755 index 5ad2ba7..0000000 --- a/utils/bin/make-chart +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 - -##### -# This script accepts a name, a target directory and a stream of manifests -# on stdin and creates an ephemeral Helm chart in the specified directory -# -# This allows Helm release semantics to be used with manifests generated by -# other tools that are missing that functionality, e.g. kustomize -# -# In particular, the Helm release semantics give us: -# * The tracking of resources across releases -# * Removal of resources that no longer exist -# * Logic for waiting for resources to become ready -##### - -import pathlib -import re -import sys - -from ruamel import yaml - - -CHART_YAML = """\ -apiVersion: v2 -name: {name} -version: 0.1.0 -""" - - -def main(name, directory): - # Make sure that the crd and templates directories exist within the chart directory - directory = pathlib.Path(directory).resolve() - crds_directory = directory / "crds" - chart_directory = directory / "chart" - crds_directory.mkdir(parents = True, exist_ok = True) - templates_directory = chart_directory / "templates" - templates_directory.mkdir(parents = True, exist_ok = True) - # Write the Chart.yaml file - chart_file = chart_directory / "Chart.yaml" - with chart_file.open("w") as f: - f.write(CHART_YAML.format(name = name)) - # For each YAML document in the stdin, write it to a separate file in the given directory - # CRDs go in the crds directory and everything else in the chart's templates directory - for document in yaml.safe_load_all(sys.stdin): - filename = "{}_{}_{}_{}.yaml".format( - document["apiVersion"].replace("/", "_"), - document["kind"].lower(), - document["metadata"].get("namespace", ""), - document["metadata"]["name"] - ) - if document["kind"] == "CustomResourceDefinition": - path = crds_directory / filename - else: - path = templates_directory / filename - # Escape any go template syntax in the resulting document - # Note that we only need to escape the starting delimiters as ending delimiters - # are ignored without the corresponding start delimiter - content = yaml.safe_dump(document, default_flow_style = False) - content = re.sub(r"\{\{\-?", "{{ \"\g<0>\" }}", content) - with path.open("w") as f: - f.write(content) - - -if __name__ == "__main__": - name, directory = sys.argv[1:] - main(name, directory)